{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a57d6663-4055-40a6-a9fc-dffb38c418b1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T07:30:53.435174Z",
     "iopub.status.busy": "2024-09-02T07:30:53.434993Z",
     "iopub.status.idle": "2024-09-02T07:30:53.437324Z",
     "shell.execute_reply": "2024-09-02T07:30:53.437001Z",
     "shell.execute_reply.started": "2024-09-02T07:30:53.435144Z"
    }
   },
   "outputs": [],
   "source": [
    "%%capture --no-stderr\n",
    "!pip install -U langchain langchain-community langchain-openai langchain-cohere pypdf sentence_transformers chromadb shutil openpyxl FlagEmbedding cohere"
   ]
  },
  {
   "cell_type": "code",
   "id": "603ae327-d834-4074-b8f0-5801669a6f26",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T07:30:53.438150Z",
     "iopub.status.busy": "2024-09-02T07:30:53.438018Z",
     "iopub.status.idle": "2024-09-02T07:30:53.451374Z",
     "shell.execute_reply": "2024-09-02T07:30:53.451021Z",
     "shell.execute_reply.started": "2024-09-02T07:30:53.438137Z"
    },
    "ExecuteTime": {
     "end_time": "2024-09-02T15:27:54.473394Z",
     "start_time": "2024-09-02T15:27:54.471190Z"
    }
   },
   "source": [
    "%env LLM_API_KEY=替换为自己的千问API Key\n",
    "%env LLM_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1\n",
    "%env COHERE_API_KEY=替换为自己的Cohere API Key"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: LLM_API_KEY=替换为自己的千问API Key\n",
      "env: LLM_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1\n",
      "env: COHERE_API_KEY=替换为自己的Cohere API Key\n"
     ]
    }
   ],
   "execution_count": 1
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1b6c2792-91bf-464a-bb0d-9724f790c6f9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T07:30:53.451864Z",
     "iopub.status.busy": "2024-09-02T07:30:53.451740Z",
     "iopub.status.idle": "2024-09-02T07:30:57.936090Z",
     "shell.execute_reply": "2024-09-02T07:30:57.935645Z",
     "shell.execute_reply.started": "2024-09-02T07:30:53.451851Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda3/lib/python3.10/site-packages/sentence_transformers/cross_encoder/CrossEncoder.py:11: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n",
      "  from tqdm.autonotebook import tqdm, trange\n",
      "2024-09-02 15:30:55.279988: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda/lib64:/usr/local/cuda/lib64:\n",
      "2024-09-02 15:30:55.280005: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.\n",
      "/opt/anaconda3/lib/python3.10/site-packages/pydantic/_internal/_config.py:341: UserWarning: Valid config keys have changed in V2:\n",
      "* 'allow_population_by_field_name' has been renamed to 'populate_by_name'\n",
      "* 'smart_union' has been removed\n",
      "  warnings.warn(message, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "langchain                     0.2.10\n",
      "langchain_core                0.2.28\n",
      "langchain_community           0.2.9\n",
      "pypdf                         4.3.1\n",
      "sentence_transformers         3.0.1\n",
      "cohere                        5.8.0\n",
      "chromadb                      0.5.4\n"
     ]
    }
   ],
   "source": [
    "import langchain, langchain_community, pypdf, sentence_transformers, chromadb, langchain_core, cohere\n",
    "\n",
    "for module in (langchain, langchain_core, langchain_community, pypdf, sentence_transformers, cohere, chromadb):\n",
    "    print(f\"{module.__name__:<30}{module.__version__}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "1e2c72b8-ee12-4130-af88-699998aa230c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T07:30:57.936903Z",
     "iopub.status.busy": "2024-09-02T07:30:57.936598Z",
     "iopub.status.idle": "2024-09-02T07:30:57.939109Z",
     "shell.execute_reply": "2024-09-02T07:30:57.938752Z",
     "shell.execute_reply.started": "2024-09-02T07:30:57.936889Z"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "841d2b02-ad06-40d2-b11f-c7adccec6ca2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T07:30:57.940452Z",
     "iopub.status.busy": "2024-09-02T07:30:57.940159Z",
     "iopub.status.idle": "2024-09-02T07:30:57.954372Z",
     "shell.execute_reply": "2024-09-02T07:30:57.953272Z",
     "shell.execute_reply.started": "2024-09-02T07:30:57.940440Z"
    }
   },
   "outputs": [],
   "source": [
    "expr_version = 'retrieval_v5_rerank'\n",
    "\n",
    "preprocess_output_dir = os.path.join(os.path.pardir, 'outputs', 'v1_20240713')\n",
    "expr_dir = os.path.join(os.path.pardir, 'experiments', expr_version)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cf7e81e3-4c82-4842-aef5-7592caaf1d39",
   "metadata": {},
   "source": [
    "# 读取文档"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "e6920e29-bc7d-4635-be06-d151eaf0e100",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T07:30:57.956096Z",
     "iopub.status.busy": "2024-09-02T07:30:57.955713Z",
     "iopub.status.idle": "2024-09-02T07:30:59.389323Z",
     "shell.execute_reply": "2024-09-02T07:30:59.388864Z",
     "shell.execute_reply.started": "2024-09-02T07:30:57.956060Z"
    }
   },
   "outputs": [],
   "source": [
    "from langchain_community.document_loaders import PyPDFLoader\n",
    "\n",
    "loader = PyPDFLoader(os.path.join(os.path.pardir, 'data', '2024全球经济金融展望报告.pdf'))\n",
    "documents = loader.load()\n",
    "\n",
    "qa_df = pd.read_excel(os.path.join(preprocess_output_dir, 'question_answer.xlsx'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "841ec659-4ad7-4e1f-b1ea-3477bf97fde3",
   "metadata": {},
   "source": [
    "# 文档切分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "74fe856a-7c19-4c3c-bb30-7abfa6298f74",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T07:30:59.390331Z",
     "iopub.status.busy": "2024-09-02T07:30:59.389872Z",
     "iopub.status.idle": "2024-09-02T07:30:59.396604Z",
     "shell.execute_reply": "2024-09-02T07:30:59.396188Z",
     "shell.execute_reply.started": "2024-09-02T07:30:59.390314Z"
    }
   },
   "outputs": [],
   "source": [
    "from uuid import uuid4\n",
    "import os\n",
    "import pickle\n",
    "\n",
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "\n",
    "def split_docs(documents, filepath, chunk_size=400, chunk_overlap=40, seperators=['\\n\\n\\n', '\\n\\n'], force_split=False):\n",
    "    if os.path.exists(filepath) and not force_split:\n",
    "        print('found cache, restoring...')\n",
    "        return pickle.load(open(filepath, 'rb'))\n",
    "\n",
    "    splitter = RecursiveCharacterTextSplitter(\n",
    "        chunk_size=chunk_size,\n",
    "        chunk_overlap=chunk_overlap,\n",
    "        separators=seperators\n",
    "    )\n",
    "    split_docs = splitter.split_documents(documents)\n",
    "    for chunk in split_docs:\n",
    "        chunk.metadata['uuid'] = str(uuid4())\n",
    "\n",
    "    pickle.dump(split_docs, open(filepath, 'wb'))\n",
    "\n",
    "    return split_docs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "aa25540d-0504-4ae7-9804-9e3862b132d5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T07:30:59.397197Z",
     "iopub.status.busy": "2024-09-02T07:30:59.397079Z",
     "iopub.status.idle": "2024-09-02T07:30:59.408361Z",
     "shell.execute_reply": "2024-09-02T07:30:59.407891Z",
     "shell.execute_reply.started": "2024-09-02T07:30:59.397185Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "found cache, restoring...\n"
     ]
    }
   ],
   "source": [
    "splitted_docs = split_docs(documents, os.path.join(preprocess_output_dir, 'split_docs.pkl'), chunk_size=500, chunk_overlap=50)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "220dbc3a-fceb-4e49-a3f1-01e16660b2a6",
   "metadata": {},
   "source": [
    "# 检索"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "8598a11c-25d8-4af1-a98b-06a8c394e261",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T07:30:59.408995Z",
     "iopub.status.busy": "2024-09-02T07:30:59.408868Z",
     "iopub.status.idle": "2024-09-02T07:30:59.423047Z",
     "shell.execute_reply": "2024-09-02T07:30:59.422623Z",
     "shell.execute_reply.started": "2024-09-02T07:30:59.408983Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "device: cuda\n"
     ]
    }
   ],
   "source": [
    "from langchain.embeddings import HuggingFaceBgeEmbeddings\n",
    "from langchain_community.vectorstores import Chroma\n",
    "import torch\n",
    "\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "print(f'device: {device}')\n",
    "\n",
    "def get_embeddings(model_path):\n",
    "    embeddings = HuggingFaceBgeEmbeddings(\n",
    "        model_name=model_path,\n",
    "        model_kwargs={'device': device},\n",
    "        encode_kwargs={'normalize_embeddings': True},\n",
    "        # show_progress=True\n",
    "        query_instruction='为这个句子生成表示以用于检索相关文章：'\n",
    "    )\n",
    "    return embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "663ef1a4-5866-4f6b-8d9d-4724f62142cb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T07:30:59.423628Z",
     "iopub.status.busy": "2024-09-02T07:30:59.423510Z",
     "iopub.status.idle": "2024-09-02T07:31:14.502713Z",
     "shell.execute_reply": "2024-09-02T07:31:14.502214Z",
     "shell.execute_reply.started": "2024-09-02T07:30:59.423616Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building prefix dict from the default dictionary ...\n",
      "Loading model from cache /tmp/jieba.cache\n",
      "Loading model cost 0.491 seconds.\n",
      "Prefix dict has been built successfully.\n"
     ]
    }
   ],
   "source": [
    "import jieba\n",
    "import shutil\n",
    "\n",
    "from tqdm.auto import tqdm\n",
    "from langchain_community.vectorstores import Chroma\n",
    "from langchain.retrievers import BM25Retriever, EnsembleRetriever\n",
    "\n",
    "# 如果已下载，可以替换为本机路径\n",
    "model_path = 'stevenluo/bge-large-zh-v1.5-ft-v4'\n",
    "embeddings = get_embeddings(model_path)\n",
    "\n",
    "persist_directory = os.path.join(expr_dir, 'chroma', 'bge')\n",
    "shutil.rmtree(persist_directory, ignore_errors=True)\n",
    "vector_db = Chroma.from_documents(\n",
    "    splitted_docs,\n",
    "    embedding=embeddings,\n",
    "    persist_directory=persist_directory\n",
    ")\n",
    "chz_cut_bm25_retriever = BM25Retriever.from_documents(splitted_docs, preprocess_func=lambda text: list(jieba.cut(text)))\n",
    "\n",
    "def build_get_ensemble_retriver_fn(weights=[0.5, 0.5]):\n",
    "    return lambda k: EnsembleRetriever(\n",
    "        retrievers=[vector_db.as_retriever(search_kwargs={'k': k}), chz_cut_bm25_retriever], weights=weights\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "566c6f3c-5777-4aa9-bc60-a3ee23050506",
   "metadata": {},
   "source": [
    "## 计算没有Reranker的准确率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "b03e3382-39e9-4932-a265-69b811041629",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T07:31:14.503462Z",
     "iopub.status.busy": "2024-09-02T07:31:14.503267Z",
     "iopub.status.idle": "2024-09-02T07:31:14.506486Z",
     "shell.execute_reply": "2024-09-02T07:31:14.506150Z",
     "shell.execute_reply.started": "2024-09-02T07:31:14.503449Z"
    }
   },
   "outputs": [],
   "source": [
    "test_df = qa_df[(qa_df['dataset'] == 'test') & (qa_df['qa_type'] == 'detailed')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "32c3ad14-b217-44aa-bdb9-909b9d559668",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T07:31:14.507137Z",
     "iopub.status.busy": "2024-09-02T07:31:14.506963Z",
     "iopub.status.idle": "2024-09-02T07:31:14.520575Z",
     "shell.execute_reply": "2024-09-02T07:31:14.520096Z",
     "shell.execute_reply.started": "2024-09-02T07:31:14.507126Z"
    }
   },
   "outputs": [],
   "source": [
    "def get_hit_stat_df(get_retriever_fn, top_k_arr=list(range(1, 9))):\n",
    "    hit_stat_data = []\n",
    "\n",
    "    for k in tqdm(top_k_arr):\n",
    "        retriever = get_retriever_fn(k)\n",
    "        for idx, row in test_df.iterrows():\n",
    "            question = row['question']\n",
    "            true_uuid = row['uuid']\n",
    "            # chunks = retrieve_fn(question, k=k)\n",
    "            chunks = retriever.get_relevant_documents(question)[:k]\n",
    "            retrieved_uuids = [doc.metadata['uuid'] for doc in chunks]\n",
    "\n",
    "            hit_stat_data.append({\n",
    "                'question': question,\n",
    "                'top_k': k,\n",
    "                'hit': int(true_uuid in retrieved_uuids),\n",
    "                'retrieved_chunks': len(chunks)\n",
    "            })\n",
    "    hit_stat_df = pd.DataFrame(hit_stat_data)\n",
    "    return hit_stat_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "a63797c7-4151-4f55-8e5d-080c34265393",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T07:31:14.521221Z",
     "iopub.status.busy": "2024-09-02T07:31:14.521060Z",
     "iopub.status.idle": "2024-09-02T07:31:32.717549Z",
     "shell.execute_reply": "2024-09-02T07:31:32.717211Z",
     "shell.execute_reply.started": "2024-09-02T07:31:14.521209Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "80de1de87a324ad29a716d252747c0cf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/8 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda3/lib/python3.10/site-packages/langchain_core/_api/deprecation.py:139: LangChainDeprecationWarning: The method `BaseRetriever.get_relevant_documents` was deprecated in langchain-core 0.1.46 and will be removed in 0.3.0. Use invoke instead.\n",
      "  warn_deprecated(\n"
     ]
    }
   ],
   "source": [
    "retriever_only_hit_stat_df = get_hit_stat_df(build_get_ensemble_retriver_fn(weights=[0.5, 0.5]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "d4890789-a44c-41de-b17f-0ff505788494",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T07:31:32.719437Z",
     "iopub.status.busy": "2024-09-02T07:31:32.719304Z",
     "iopub.status.idle": "2024-09-02T07:31:32.725844Z",
     "shell.execute_reply": "2024-09-02T07:31:32.725533Z",
     "shell.execute_reply.started": "2024-09-02T07:31:32.719424Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>top_k</th>\n",
       "      <th>hit_rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.548387</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0.752688</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0.870968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>0.892473</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0.913978</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6</td>\n",
       "      <td>0.935484</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7</td>\n",
       "      <td>0.946237</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8</td>\n",
       "      <td>0.967742</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   top_k  hit_rate\n",
       "0      1  0.548387\n",
       "1      2  0.752688\n",
       "2      3  0.870968\n",
       "3      4  0.892473\n",
       "4      5  0.913978\n",
       "5      6  0.935484\n",
       "6      7  0.946237\n",
       "7      8  0.967742"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "retriever_only_hit_stat_df.groupby(['top_k'])['hit'].mean().reset_index().rename(columns={'hit': 'hit_rate'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "b0b086d1-6cec-4743-8df6-2ab3b1593689",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T07:31:32.726378Z",
     "iopub.status.busy": "2024-09-02T07:31:32.726255Z",
     "iopub.status.idle": "2024-09-02T07:31:33.059379Z",
     "shell.execute_reply": "2024-09-02T07:31:33.058884Z",
     "shell.execute_reply.started": "2024-09-02T07:31:32.726366Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: xlabel='top_k', ylabel='hit'>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAGxCAYAAACeKZf2AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/TGe4hAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAiiUlEQVR4nO3dfVSUdf7/8deAApqKmXIjomg34i2YJIvmdkex5rE4nW1Zc5XQ7FRQ6PxqFW8gM8XaJD1lkvf+djN12yx3Ncr4heZKoSitbqmZGqwK6umbKCmsM/P7o9Ps8hUNFOYaPjwf58w5zcV1Oe/PelyfXtc1MzaXy+USAACAIXysHgAAAKAxETcAAMAoxA0AADAKcQMAAIxC3AAAAKMQNwAAwCjEDQAAMApxAwAAjNLK6gE8zel06vjx42rfvr1sNpvV4wAAgHpwuVw6e/asunbtKh+fK5+baXFxc/z4cYWHh1s9BgAAuAplZWXq1q3bFfexNG62bdumP/zhDyouLtaJEye0YcMGJSYmXvGYgoIC2e12/fOf/1R4eLhmzJihRx99tN6v2b59e0k//o/ToUOHa5geAAB4SmVlpcLDw91/j1+JpXFTVVWlqKgojR8/Xg899NDP7n/kyBGNHDlSTzzxhN566y3l5+frscceU2hoqBISEur1mj9diurQoQNxAwBAM1OfW0osjZsRI0ZoxIgR9d4/NzdXPXv21Pz58yVJffr00fbt2/Xqq6/WO24AAIDZmtW7pQoLCxUfH19rW0JCggoLCy2aCAAAeJtmdUNxeXm5goODa20LDg5WZWWlzp8/rzZt2lxyTHV1taqrq93PKysrm3xOAABgnWZ15uZqZGdnKzAw0P3gnVIAAJitWcVNSEiIKioqam2rqKhQhw4d6jxrI0kZGRk6c+aM+1FWVuaJUQEAgEWa1WWpuLg4bd68uda2LVu2KC4u7rLH+Pv7y9/fv6lHAwAAXsLSMzfnzp1TSUmJSkpKJP34Vu+SkhKVlpZK+vGsy7hx49z7P/HEEzp8+LB+//vfa//+/XrjjTe0fv16TZ482YrxAQCAF7I0bnbt2qVBgwZp0KBBkiS73a5BgwYpMzNTknTixAl36EhSz549tWnTJm3ZskVRUVGaP3++li1bxtvAAQCAm83lcrmsHsKTKisrFRgYqDNnzvAhfgAANBMN+fu7Wd1QDAAA8HOIGwAAYBTiBgAAGIW4AQAARiFuAACAUYgbAABglGb1CcUAAODynn/+eatHaBTXug7O3AAAAKMQNwAAwCjEDQAAMApxAwAAjELcAAAAoxA3AADAKLwVHABgnK/m/D+rR2gUfabfbfUIzRJnbgAAgFGIGwAAYBTiBgAAGIV7bgDAYHN+92urR2gU0//0jtUjoBnhzA0AADAKcQMAAIxC3AAAAKMQNwAAwCjEDQAAMApxAwAAjELcAAAAo/A5NwBahNf/z1+tHqFRpM0fZfUIgNfjzA0AADAKcQMAAIxC3AAAAKMQNwAAwCjEDQAAMApxAwAAjELcAAAAo/A5N0ALs/WXd1g9QqO4Y9tWq0cA4KU4cwMAAIxC3AAAAKMQNwAAwCjEDQAAMApxAwAAjELcAAAAoxA3AADAKHzODVqsYa8Ns3qERvH3p/9u9QgA4FU4cwMAAIxC3AAAAKMQNwAAwCjEDQAAMApxAwAAjELcAAAAoxA3AADAKMQNAAAwCnEDAACMQtwAAACjEDcAAMAoxA0AADAKcQMAAIxC3AAAAKMQNwAAwCjEDQAAMApxAwAAjELcAAAAoxA3AADAKMQNAAAwiuVxs2jRIkVERCggIECxsbEqKiq64v4LFixQ79691aZNG4WHh2vy5Mm6cOGCh6YFAADeztK4Wbdunex2u7KysrR7925FRUUpISFBJ0+erHP/NWvWaOrUqcrKytJXX32l5cuXa926dZo2bZqHJwcAAN7K0rjJycnRxIkTlZKSor59+yo3N1dt27bVihUr6tx/x44dGjZsmB555BFFRETovvvu0+jRo3/2bA8AAGg5LIubmpoaFRcXKz4+/j/D+PgoPj5ehYWFdR4zdOhQFRcXu2Pm8OHD2rx5s+6//36PzAwAALxfK6te+PTp03I4HAoODq61PTg4WPv376/zmEceeUSnT5/W7bffLpfLpYsXL+qJJ5644mWp6upqVVdXu59XVlY2zgIAAIBXsvyG4oYoKCjQ3Llz9cYbb2j37t169913tWnTJs2ePfuyx2RnZyswMND9CA8P9+DEAADA0yw7c9O5c2f5+vqqoqKi1vaKigqFhITUeczMmTM1duxYPfbYY5KkAQMGqKqqSo8//rimT58uH59LWy0jI0N2u939vLKyksABAMBglp258fPz0+DBg5Wfn+/e5nQ6lZ+fr7i4uDqP+eGHHy4JGF9fX0mSy+Wq8xh/f3916NCh1gMAAJjLsjM3kmS325WcnKyYmBgNGTJECxYsUFVVlVJSUiRJ48aNU1hYmLKzsyVJo0aNUk5OjgYNGqTY2FgdOnRIM2fO1KhRo9yRAwAAWjZL4yYpKUmnTp1SZmamysvLFR0drby8PPdNxqWlpbXO1MyYMUM2m00zZszQsWPH1KVLF40aNUpz5syxagkAAMDLWBo3kpSWlqa0tLQ6f1ZQUFDreatWrZSVlaWsrCwPTAYAAJqjZvVuKQAAgJ9D3AAAAKMQNwAAwCjEDQAAMApxAwAAjELcAAAAoxA3AADAKMQNAAAwCnEDAACMQtwAAACjEDcAAMAoln+3FKxX+sIAq0doFN0z91o9AgDAC3DmBgAAGIW4AQAARiFuAACAUYgbAABgFOIGAAAYhbgBAABGIW4AAIBRiBsAAGAU4gYAABiFuAEAAEYhbgAAgFGIGwAAYBTiBgAAGIW4AQAARiFuAACAUYgbAABgFOIGAAAYhbgBAABGIW4AAIBRiBsAAGAU4gYAABiFuAEAAEYhbgAAgFGIGwAAYBTiBgAAGIW4AQAARiFuAACAUYgbAABgFOIGAAAYhbgBAABGIW4AAIBRiBsAAGAU4gYAABiFuAEAAEYhbgAAgFGIGwAAYBTiBgAAGIW4AQAARiFuAACAUYgbAABgFOIGAAAYhbgBAABGIW4AAIBRiBsAAGAU4gYAABiFuAEAAEYhbgAAgFGIGwAAYBTiBgAAGIW4AQAARrE8bhYtWqSIiAgFBAQoNjZWRUVFV9z/+++/V2pqqkJDQ+Xv769bbrlFmzdv9tC0AADA27Wy8sXXrVsnu92u3NxcxcbGasGCBUpISNCBAwcUFBR0yf41NTW69957FRQUpHfeeUdhYWH69ttv1bFjR88PDwAAvJKlcZOTk6OJEycqJSVFkpSbm6tNmzZpxYoVmjp16iX7r1ixQt9995127Nih1q1bS5IiIiI8OTIAAPByll2WqqmpUXFxseLj4/8zjI+P4uPjVVhYWOcxGzduVFxcnFJTUxUcHKz+/ftr7ty5cjgcl32d6upqVVZW1noAAABzWRY3p0+flsPhUHBwcK3twcHBKi8vr/OYw4cP65133pHD4dDmzZs1c+ZMzZ8/Xy+++OJlXyc7O1uBgYHuR3h4eKOuAwAAeBfLbyhuCKfTqaCgIC1ZskSDBw9WUlKSpk+frtzc3Msek5GRoTNnzrgfZWVlHpwYAAB4mmX33HTu3Fm+vr6qqKiotb2iokIhISF1HhMaGqrWrVvL19fXva1Pnz4qLy9XTU2N/Pz8LjnG399f/v7+jTs8AADwWpadufHz89PgwYOVn5/v3uZ0OpWfn6+4uLg6jxk2bJgOHTokp9Pp3nbw4EGFhobWGTYAAKDlsfSylN1u19KlS7V69Wp99dVXevLJJ1VVVeV+99S4ceOUkZHh3v/JJ5/Ud999p/T0dB08eFCbNm3S3LlzlZqaatUSAACAl7H0reBJSUk6deqUMjMzVV5erujoaOXl5blvMi4tLZWPz3/6Kzw8XB9++KEmT56sgQMHKiwsTOnp6ZoyZYpVSwAAAF7G0riRpLS0NKWlpdX5s4KCgku2xcXF6bPPPmviqQAAQHPVrN4tBQAA8HOIGwAAYBTiBgAAGIW4AQAARiFuAACAUYgbAABgFOIGAAAYhbgBAABGIW4AAIBRiBsAAGAU4gYAABiFuAEAAEax/Iszvcng5/6v1SM0iuI/jLN6BAAALMOZGwAAYBTiBgAAGIW4AQAARiFuAACAUYgbAABgFOIGAAAYhbgBAABGIW4AAIBRripu7r77bn3//feXbK+srNTdd999rTMBAABctauKm4KCAtXU1Fyy/cKFC/r000+veSgAAICr1aCvX/jHP/7h/u8vv/xS5eXl7ucOh0N5eXkKCwtrvOkAAAAaqEFxEx0dLZvNJpvNVuflpzZt2ui1115rtOEAAAAaqkFxc+TIEblcLvXq1UtFRUXq0qWL+2d+fn4KCgqSr69vow8JAABQXw2Kmx49ekiSnE5nkwwDAABwreodNxs3btSIESPUunVrbdy48Yr7PvDAA9c8GAAAwNWod9wkJiaqvLxcQUFBSkxMvOx+NptNDoejMWYDAABosHrHzX9fiuKyFAAA8FYNuufmv+Xn5ys/P18nT56sFTs2m03Lly9vlOEAAAAa6qriZtasWXrhhRcUExOj0NBQ2Wy2xp4LAADgqlxV3OTm5mrVqlUaO3ZsY88DAABwTa7q6xdqamo0dOjQxp4FAADgml1V3Dz22GNas2ZNY88CAABwzep9Wcput7v/2+l0asmSJfr44481cOBAtW7duta+OTk5jTchAABAA9Q7bvbs2VPreXR0tCRp3759tbZzczEAALBSvePmk08+aco5AAAAGsVV3XMDAADgrYgbAABgFOIGAAAYhbgBAABGIW4AAIBRiBsAAGAU4gYAABiFuAEAAEYhbgAAgFGIGwAAYBTiBgAAGIW4AQAARiFuAACAUYgbAABgFOIGAAAYhbgBAABGIW4AAIBRiBsAAGAU4gYAABiFuAEAAEYhbgAAgFGIGwAAYBTiBgAAGIW4AQAARvGKuFm0aJEiIiIUEBCg2NhYFRUV1eu4tWvXymazKTExsWkHBAAAzYblcbNu3TrZ7XZlZWVp9+7dioqKUkJCgk6ePHnF444ePapnn31Ww4cP99CkAACgObA8bnJycjRx4kSlpKSob9++ys3NVdu2bbVixYrLHuNwODRmzBjNmjVLvXr18uC0AADA21kaNzU1NSouLlZ8fLx7m4+Pj+Lj41VYWHjZ41544QUFBQVpwoQJP/sa1dXVqqysrPUAAADmsjRuTp8+LYfDoeDg4Frbg4ODVV5eXucx27dv1/Lly7V06dJ6vUZ2drYCAwPdj/Dw8GueGwAAeC/LL0s1xNmzZzV27FgtXbpUnTt3rtcxGRkZOnPmjPtRVlbWxFMCAAArtbLyxTt37ixfX19VVFTU2l5RUaGQkJBL9v/mm2909OhRjRo1yr3N6XRKklq1aqUDBw7oxhtvrHWMv7+//P39m2B6AADgjSw9c+Pn56fBgwcrPz/fvc3pdCo/P19xcXGX7B8ZGam9e/eqpKTE/XjggQd01113qaSkhEtOAADA2jM3kmS325WcnKyYmBgNGTJECxYsUFVVlVJSUiRJ48aNU1hYmLKzsxUQEKD+/fvXOr5jx46SdMl2AADQMlkeN0lJSTp16pQyMzNVXl6u6Oho5eXluW8yLi0tlY9Ps7o1CAAAWMjyuJGktLQ0paWl1fmzgoKCKx67atWqxh8IAAA0W5wSAQAARiFuAACAUYgbAABgFOIGAAAYhbgBAABGIW4AAIBRiBsAAGAU4gYAABiFuAEAAEYhbgAAgFGIGwAAYBTiBgAAGIW4AQAARiFuAACAUYgbAABgFOIGAAAYhbgBAABGIW4AAIBRiBsAAGAU4gYAABiFuAEAAEYhbgAAgFGIGwAAYBTiBgAAGIW4AQAARiFuAACAUYgbAABgFOIGAAAYhbgBAABGIW4AAIBRiBsAAGAU4gYAABiFuAEAAEYhbgAAgFGIGwAAYBTiBgAAGIW4AQAARiFuAACAUYgbAABgFOIGAAAYhbgBAABGIW4AAIBRiBsAAGAU4gYAABiFuAEAAEYhbgAAgFGIGwAAYBTiBgAAGIW4AQAARiFuAACAUYgbAABgFOIGAAAYhbgBAABGIW4AAIBRiBsAAGAU4gYAABiFuAEAAEYhbgAAgFGIGwAAYBTiBgAAGIW4AQAARvGKuFm0aJEiIiIUEBCg2NhYFRUVXXbfpUuXavjw4br++ut1/fXXKz4+/or7AwCAlsXyuFm3bp3sdruysrK0e/duRUVFKSEhQSdPnqxz/4KCAo0ePVqffPKJCgsLFR4ervvuu0/Hjh3z8OQAAMAbWR43OTk5mjhxolJSUtS3b1/l5uaqbdu2WrFiRZ37v/XWW3rqqacUHR2tyMhILVu2TE6nU/n5+R6eHAAAeCNL46ampkbFxcWKj493b/Px8VF8fLwKCwvr9Wv88MMP+ve//61OnTo11ZgAAKAZaWXli58+fVoOh0PBwcG1tgcHB2v//v31+jWmTJmirl271gqk/1ZdXa3q6mr388rKyqsfGAAAeD3LL0tdi3nz5mnt2rXasGGDAgIC6twnOztbgYGB7kd4eLiHpwQAAJ5kadx07txZvr6+qqioqLW9oqJCISEhVzz2lVde0bx58/TRRx9p4MCBl90vIyNDZ86ccT/KysoaZXYAAOCdLI0bPz8/DR48uNbNwD/dHBwXF3fZ415++WXNnj1beXl5iomJueJr+Pv7q0OHDrUeAADAXJbecyNJdrtdycnJiomJ0ZAhQ7RgwQJVVVUpJSVFkjRu3DiFhYUpOztbkvTSSy8pMzNTa9asUUREhMrLyyVJ7dq1U7t27SxbBwAA8A6Wx01SUpJOnTqlzMxMlZeXKzo6Wnl5ee6bjEtLS+Xj858TTIsXL1ZNTY1+/etf1/p1srKy9Pzzz3tydAAA4IUsjxtJSktLU1paWp0/KygoqPX86NGjTT8QAABotpr1u6UAAAD+N+IGAAAYhbgBAABGIW4AAIBRiBsAAGAU4gYAABiFuAEAAEYhbgAAgFGIGwAAYBTiBgAAGIW4AQAARiFuAACAUYgbAABgFOIGAAAYhbgBAABGIW4AAIBRiBsAAGAU4gYAABiFuAEAAEYhbgAAgFGIGwAAYBTiBgAAGIW4AQAARiFuAACAUYgbAABgFOIGAAAYhbgBAABGIW4AAIBRiBsAAGAU4gYAABiFuAEAAEYhbgAAgFGIGwAAYBTiBgAAGIW4AQAARiFuAACAUYgbAABgFOIGAAAYhbgBAABGIW4AAIBRiBsAAGAU4gYAABiFuAEAAEYhbgAAgFGIGwAAYBTiBgAAGIW4AQAARiFuAACAUYgbAABgFOIGAAAYhbgBAABGIW4AAIBRiBsAAGAU4gYAABiFuAEAAEYhbgAAgFGIGwAAYBTiBgAAGIW4AQAARiFuAACAUYgbAABgFOIGAAAYxSviZtGiRYqIiFBAQIBiY2NVVFR0xf3//Oc/KzIyUgEBARowYIA2b97soUkBAIC3szxu1q1bJ7vdrqysLO3evVtRUVFKSEjQyZMn69x/x44dGj16tCZMmKA9e/YoMTFRiYmJ2rdvn4cnBwAA3sjyuMnJydHEiROVkpKivn37Kjc3V23bttWKFSvq3H/hwoX61a9+peeee059+vTR7Nmzdeutt+r111/38OQAAMAbWRo3NTU1Ki4uVnx8vHubj4+P4uPjVVhYWOcxhYWFtfaXpISEhMvuDwAAWpZWVr746dOn5XA4FBwcXGt7cHCw9u/fX+cx5eXlde5fXl5e5/7V1dWqrq52Pz9z5owkqbKy8pJ9HdXnGzS/t6prbVdy9oKjiSbxrIau++L5i000iWc1dN1VF1vmus9X/9BEk3hWQ9d94d//bqJJPKuh6z53oaqJJvGshq77v/++a87qWvdP21wu188eb2nceEJ2drZmzZp1yfbw8HALpvGMwNeesHoEa2QHWj2BJQKntMx1K7Blrvv3i6yewBovrm+Zv9960eoBrDFv3rzL/uzs2bMK/Jk//5bGTefOneXr66uKiopa2ysqKhQSElLnMSEhIQ3aPyMjQ3a73f3c6XTqu+++0w033CCbzXaNK2iYyspKhYeHq6ysTB06dPDoa1uJdbPuloB1s+6WwMp1u1wunT17Vl27dv3ZfS2NGz8/Pw0ePFj5+flKTEyU9GN85OfnKy0trc5j4uLilJ+fr0mTJrm3bdmyRXFxcXXu7+/vL39//1rbOnbs2BjjX7UOHTq0qD8MP2HdLQvrbllYd8ti1bp/7ozNTyy/LGW325WcnKyYmBgNGTJECxYsUFVVlVJSUiRJ48aNU1hYmLKzsyVJ6enpuuOOOzR//nyNHDlSa9eu1a5du7RkyRIrlwEAALyE5XGTlJSkU6dOKTMzU+Xl5YqOjlZeXp77puHS0lL5+PznTV1Dhw7VmjVrNGPGDE2bNk0333yz3nvvPfXv39+qJQAAAC9iedxIUlpa2mUvQxUUFFyy7eGHH9bDDz/cxFM1Pn9/f2VlZV1ymcx0rJt1twSsm3W3BM1l3TZXfd5TBQAA0ExY/gnFAAAAjYm4AQAARiFuAACAUYgbD9i2bZtGjRqlrl27ymaz6b333rN6JI/Izs7Wbbfdpvbt2ysoKEiJiYk6cOCA1WM1ucWLF2vgwIHuz4GIi4vTBx98YPVYHjdv3jzZbLZan0lloueff142m63WIzIy0uqxPOLYsWP63e9+pxtuuEFt2rTRgAEDtGvXLqvHalIRERGX/H7bbDalpqZaPVqTcjgcmjlzpnr27Kk2bdroxhtv1OzZs+v1VQhW8Ip3S5muqqpKUVFRGj9+vB566CGrx/GYrVu3KjU1VbfddpsuXryoadOm6b777tOXX36p6667zurxmky3bt00b9483XzzzXK5XFq9erUefPBB7dmzR/369bN6PI/YuXOn3nzzTQ0cONDqUTyiX79++vjjj93PW7Uy//9a/+d//kfDhg3TXXfdpQ8++EBdunTR119/reuvv97q0ZrUzp075XD85/v49u3bp3vvvbdZvoO3IV566SUtXrxYq1evVr9+/bRr1y6lpKQoMDBQzzzzjNXjXcL8P4FeYMSIERoxYoTVY3hcXl5ereerVq1SUFCQiouL9ctf/tKiqZreqFGjaj2fM2eOFi9erM8++6xFxM25c+c0ZswYLV26VC++2DK+GKdVq1aX/QoYU7300ksKDw/XypUr3dt69uxp4USe0aVLl1rP582bpxtvvFF33HGHRRN5xo4dO/Tggw9q5MiRkn48g/X222+rqKjI4snqxmUpeMxP38jeqVMniyfxHIfDobVr16qqquqyXxFimtTUVI0cOVLx8fFWj+IxX3/9tbp27apevXppzJgxKi0ttXqkJrdx40bFxMTo4YcfVlBQkAYNGqSlS5daPZZH1dTU6E9/+pPGjx/v8e8q9LShQ4cqPz9fBw8elCR98cUX2r59u9f+w50zN/AIp9OpSZMmadiwYS3i06T37t2ruLg4XbhwQe3atdOGDRvUt29fq8dqcmvXrtXu3bu1c+dOq0fxmNjYWK1atUq9e/fWiRMnNGvWLA0fPlz79u1T+/btrR6vyRw+fFiLFy+W3W7XtGnTtHPnTj3zzDPy8/NTcnKy1eN5xHvvvafvv/9ejz76qNWjNLmpU6eqsrJSkZGR8vX1lcPh0Jw5czRmzBirR6sTcQOPSE1N1b59+7R9+3arR/GI3r17q6SkRGfOnNE777yj5ORkbd261ejAKSsrU3p6urZs2aKAgACrx/GY//6X68CBAxUbG6sePXpo/fr1mjBhgoWTNS2n06mYmBjNnTtXkjRo0CDt27dPubm5LSZuli9frhEjRtTrW6qbu/Xr1+utt97SmjVr1K9fP5WUlGjSpEnq2rWrV/5+Ezdocmlpafrb3/6mbdu2qVu3blaP4xF+fn666aabJEmDBw/Wzp07tXDhQr355psWT9Z0iouLdfLkSd16663ubQ6HQ9u2bdPrr7+u6upq+fr6WjihZ3Ts2FG33HKLDh06ZPUoTSo0NPSSWO/Tp4/+8pe/WDSRZ3377bf6+OOP9e6771o9ikc899xzmjp1qn77299KkgYMGKBvv/1W2dnZxA1aFpfLpaefflobNmxQQUFBi7jZ8HKcTqeqq6utHqNJ3XPPPdq7d2+tbSkpKYqMjNSUKVNaRNhIP95Q/c0332js2LFWj9Kkhg0bdslHOxw8eFA9evSwaCLPWrlypYKCgtw32Jruhx9+qPUl1pLk6+srp9Np0URXRtx4wLlz52r9K+7IkSMqKSlRp06d1L17dwsna1qpqalas2aN3n//fbVv317l5eWSpMDAQLVp08bi6ZpORkaGRowYoe7du+vs2bNas2aNCgoK9OGHH1o9WpNq3779JfdTXXfddbrhhhuMvs/q2Wef1ahRo9SjRw8dP35cWVlZ8vX11ejRo60erUlNnjxZQ4cO1dy5c/Wb3/xGRUVFWrJkiZYsWWL1aE3O6XRq5cqVSk5ObhFv+5d+fBfonDlz1L17d/Xr10979uxRTk6Oxo8fb/VodXOhyX3yyScuSZc8kpOTrR6tSdW1ZkmulStXWj1akxo/fryrR48eLj8/P1eXLl1c99xzj+ujjz6yeixL3HHHHa709HSrx2hSSUlJrtDQUJefn58rLCzMlZSU5Dp06JDVY3nEX//6V1f//v1d/v7+rsjISNeSJUusHskjPvzwQ5ck14EDB6wexWMqKytd6enpru7du7sCAgJcvXr1ck2fPt1VXV1t9Wh14lvBAQCAUficGwAAYBTiBgAAGIW4AQAARiFuAACAUYgbAABgFOIGAAAYhbgBAABGIW4AAIBRiBsALVpERIQWLFhg9RgAGhFxA8Br3HnnnZo0aZLVYwBo5ogbAABgFOIGgFd49NFHtXXrVi1cuFA2m002m01Hjx7V1q1bNWTIEPn7+ys0NFRTp07VxYsX3cfdeeedSktLU1pamgIDA9W5c2fNnDlTV/u1ecuWLVPHjh2Vn5/fWEsD4GHEDQCvsHDhQsXFxWnixIk6ceKETpw4odatW+v+++/Xbbfdpi+++EKLFy/W8uXL9eKLL9Y6dvXq1WrVqpWKioq0cOFC5eTkaNmyZQ2e4eWXX9bUqVP10Ucf6Z577mmspQHwsFZWDwAAkhQYGCg/Pz+1bdtWISEhkqTp06crPDxcr7/+umw2myIjI3X8+HFNmTJFmZmZ8vH58d9n4eHhevXVV2Wz2dS7d2/t3btXr776qiZOnFjv158yZYr++Mc/auvWrerXr1+TrBGAZ3DmBoDX+uqrrxQXFyebzebeNmzYMJ07d07/+te/3Nt+8Ytf1NonLi5OX3/9tRwOR71eZ/78+Vq6dKm2b99O2AAGIG4AtHjDhw+Xw+HQ+vXrrR4FQCMgbgB4DT8/v1pnW/r06aPCwsJaNwf//e9/V/v27dWtWzf3ts8//7zWr/PZZ5/p5ptvlq+vb71ed8iQIfrggw80d+5cvfLKK9e4CgBWI24AeI2IiAh9/vnnOnr0qE6fPq2nnnpKZWVlevrpp7V//369//77ysrKkt1ud99vI0mlpaWy2+06cOCA3n77bb322mtKT09v0GsPHTpUmzdv1qxZs/hQP6CZ44ZiAF7j2WefVXJysvr27avz58/ryJEj2rx5s5577jlFRUWpU6dOmjBhgmbMmFHruHHjxun8+fMaMmSIfH19lZ6erscff7zBr3/77bdr06ZNuv/+++Xr66unn366sZYGwINsrqv9MAgA8AJ33nmnoqOjOdsCwI3LUgAAwCjEDQBjffrpp2rXrt1lHwDMxGUpAMY6f/68jh07dtmf33TTTR6cBoCnEDcAAMAoXJYCAABGIW4AAIBRiBsAAGAU4gYAABiFuAEAAEYhbgAAgFGIGwAAYBTiBgAAGOX/A9ji3fZw1hkGAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "\n",
    "sns.barplot(x='top_k', y='hit', data=retriever_only_hit_stat_df, errorbar=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e6c2e518-ae0c-42e6-b610-9cc68cb68448",
   "metadata": {},
   "source": [
    "## 加Reranker"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3dc58af5-bd58-4719-9ca1-9652d5790c74",
   "metadata": {},
   "source": [
    "### Coheree Rerank"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "034746a0-028d-41cf-98f4-b2a739994cfd",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-15T03:17:39.918059Z",
     "iopub.status.busy": "2024-08-15T03:17:39.917257Z",
     "iopub.status.idle": "2024-08-15T03:17:39.936058Z",
     "shell.execute_reply": "2024-08-15T03:17:39.933046Z",
     "shell.execute_reply.started": "2024-08-15T03:17:39.917988Z"
    }
   },
   "source": [
    "官方样例\n",
    "\n",
    "```python\n",
    "import cohere\n",
    "\n",
    "co = cohere.Client(\"<<apiKey>>\")\n",
    "\n",
    "docs = [\n",
    "    \"Carson City is the capital city of the American state of Nevada.\",\n",
    "    \"The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean. Its capital is Saipan.\",\n",
    "    \"Capitalization or capitalisation in English grammar is the use of a capital letter at the start of a word. English usage varies from capitalization in other languages.\",\n",
    "    \"Washington, D.C. (also known as simply Washington or D.C., and officially as the District of Columbia) is the capital of the United States. It is a federal district.\",\n",
    "    \"Capital punishment (the death penalty) has existed in the United States since beforethe United States was a country. As of 2017, capital punishment is legal in 30 of the 50 states.\",\n",
    "]\n",
    "\n",
    "response = co.rerank(\n",
    "    model=\"rerank-multilingual-v3.0\",\n",
    "    query=\"What is the capital of the United States?\",\n",
    "    documents=docs,\n",
    "    top_n=3,\n",
    ")\n",
    "print(response)\n",
    "```\n",
    "\n",
    "```bash\n",
    "curl --request POST \\\n",
    "  --url https://api.cohere.com/v1/rerank \\\n",
    "  --header 'accept: application/json' \\\n",
    "  --header 'content-type: application/json' \\\n",
    "  --header \"Authorization: bearer <api_key>\" \\\n",
    "  --data '{\n",
    "    \"model\": \"rerank-english-v3.0\",\n",
    "    \"query\": \"What is the capital of the United States?\",\n",
    "    \"top_n\": 3,\n",
    "    \"documents\": [\"Carson City is the capital city of the American state of Nevada.\",\n",
    "                  \"The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean. Its capital is Saipan.\",\n",
    "                  \"Washington, D.C. (also known as simply Washington or D.C., and officially as the District of Columbia) is the capital of the United States. It is a federal district.\",\n",
    "                  \"Capitalization or capitalisation in English grammar is the use of a capital letter at the start of a word. English usage varies from capitalization in other languages.\",\n",
    "                  \"Capital punishment (the death penalty) has existed in the United States since beforethe United States was a country. As of 2017, capital punishment is legal in 30 of the 50 states.\"]\n",
    "```\n",
    "resp\n",
    "```json\n",
    "{\"id\":\"0b91f9b5-fb76-468c-977b-e403c7cc4113\",\"results\":[{\"index\":2,\"relevance_score\":0.999071},{\"index\":4,\"relevance_score\":0.7867867},{\"index\":0,\"relevance_score\":0.32713068}],\"meta\":{\"api_version\":{\"version\":\"1\"},\"billed_units\":{\"search_units\":1}}}\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "908b50f3-5e59-41b2-823d-9894ae8d683a",
   "metadata": {},
   "source": [
    "使用Cohere Rerank在线API服务，需要有代理服务"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "2b35c45a-9897-41c6-8f9d-98410c12c82c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T07:31:33.060149Z",
     "iopub.status.busy": "2024-09-02T07:31:33.059849Z",
     "iopub.status.idle": "2024-09-02T07:31:33.062856Z",
     "shell.execute_reply": "2024-09-02T07:31:33.062544Z",
     "shell.execute_reply.started": "2024-09-02T07:31:33.060135Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: http_proxy=http://192.168.31.71:4780\n",
      "env: https_proxy=http://192.168.31.71:4780\n"
     ]
    }
   ],
   "source": [
    "%env http_proxy=http://192.168.31.71:4780\n",
    "%env https_proxy=http://192.168.31.71:4780"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "851b193b-9bb0-4201-b0d4-467ddd32400c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T07:31:33.063391Z",
     "iopub.status.busy": "2024-09-02T07:31:33.063268Z",
     "iopub.status.idle": "2024-09-02T07:31:33.276853Z",
     "shell.execute_reply": "2024-09-02T07:31:33.276393Z",
     "shell.execute_reply.started": "2024-09-02T07:31:33.063379Z"
    }
   },
   "outputs": [],
   "source": [
    "from langchain_cohere import CohereRerank\n",
    "\n",
    "cohere_rerank = CohereRerank(model=\"rerank-english-v3.0\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4015ea9f-d8e6-4488-acd7-552fd96bc983",
   "metadata": {},
   "source": [
    "试用一下"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "e6c1218b-1b2a-4e9f-a7c4-6558dfbffce9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T07:31:33.277530Z",
     "iopub.status.busy": "2024-09-02T07:31:33.277378Z",
     "iopub.status.idle": "2024-09-02T07:31:33.279926Z",
     "shell.execute_reply": "2024-09-02T07:31:33.279609Z",
     "shell.execute_reply.started": "2024-09-02T07:31:33.277517Z"
    }
   },
   "outputs": [],
   "source": [
    "# API用量用完了\n",
    "\n",
    "# docs = [\n",
    "#     \"Carson City is the capital city of the American state of Nevada.\",\n",
    "#     \"The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean. Its capital is Saipan.\",\n",
    "#     \"Capitalization or capitalisation in English grammar is the use of a capital letter at the start of a word. English usage varies from capitalization in other languages.\",\n",
    "#     \"Washington, D.C. (also known as simply Washington or D.C., and officially as the District of Columbia) is the capital of the United States. It is a federal district.\",\n",
    "#     \"Capital punishment (the death penalty) has existed in the United States since beforethe United States was a country. As of 2017, capital punishment is legal in 30 of the 50 states.\",\n",
    "# ]\n",
    "# query = \"What is the capital of the United States?\"\n",
    "# cohere_rerank.rerank(\n",
    "#     documents=docs,\n",
    "#     query=query,\n",
    "#     top_n=None\n",
    "# )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "922105b4-882e-4063-8414-f18f2cc40a50",
   "metadata": {},
   "source": [
    "重新封装一下，以便统一本地和Cohere的Rerank"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "00910605-83f9-470e-94cb-4f5f9cf3115f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T07:31:33.280507Z",
     "iopub.status.busy": "2024-09-02T07:31:33.280388Z",
     "iopub.status.idle": "2024-09-02T07:31:33.294394Z",
     "shell.execute_reply": "2024-09-02T07:31:33.294008Z",
     "shell.execute_reply.started": "2024-09-02T07:31:33.280496Z"
    }
   },
   "outputs": [],
   "source": [
    "import time\n",
    "\n",
    "class LangchainCohereRerank:\n",
    "    def __init__(self, api_key=None, model='rerank-multilingual-v3.0'):\n",
    "        if api_key is not None:\n",
    "            os.environ['COHERE_API_KEY'] = api_key\n",
    "        if os.environ.get('COHERE_API_KEY') is None:\n",
    "            raise ValueError('api_key not provided! export COHERE_API_KEY environment variable os pass api_key during construct!')\n",
    "            \n",
    "        self.cohere_rerank = CohereRerank(model=model)\n",
    "\n",
    "    def compute_score(self, query_documents, max_retry=5):\n",
    "        query_docs_dict = {}\n",
    "        for query, doc in query_documents:\n",
    "            if query not in query_docs_dict:\n",
    "                query_docs_dict[query] = []\n",
    "            query_docs_dict[query].append(doc)\n",
    "        \n",
    "        ret = []\n",
    "        for query, docs in query_docs_dict.items():\n",
    "            scores = [0] * len(docs)\n",
    "            \"\"\"\n",
    "            resp样例：\n",
    "            [{'index': 3, 'relevance_score': 0.999071},\n",
    "             {'index': 4, 'relevance_score': 0.7867867},\n",
    "             {'index': 0, 'relevance_score': 0.32713068}]\n",
    "            \"\"\"\n",
    "            while max_retry > 0:\n",
    "                try:\n",
    "                    resp = cohere_rerank.rerank(\n",
    "                        query=query,\n",
    "                        documents=docs,\n",
    "                        top_n=None\n",
    "                    )\n",
    "                    for item in resp:\n",
    "                        scores[item['index']] = item['relevance_score']\n",
    "                    ret.extend(scores)\n",
    "                    break\n",
    "                except Exception as e:\n",
    "                    max_retry -= 1\n",
    "                    sleep_s = 20 + 2 ** (8 - max_retry)\n",
    "                    print(f\"query: {query}, error: {e} doc count: {len(docs)}, sleeping {sleep_s}s, retry left {max_retry}\")\n",
    "                    time.sleep(sleep_s)\n",
    "        return ret"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "f950d998-e521-4c88-9654-241d7b59f71a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T07:31:33.295029Z",
     "iopub.status.busy": "2024-09-02T07:31:33.294852Z",
     "iopub.status.idle": "2024-09-02T07:31:33.301558Z",
     "shell.execute_reply": "2024-09-02T07:31:33.301122Z",
     "shell.execute_reply.started": "2024-09-02T07:31:33.295016Z"
    }
   },
   "outputs": [],
   "source": [
    "# API用量用完了\n",
    "\n",
    "# LangchainCohereRerank().compute_score(\n",
    "#     [[query, doc] for doc in docs]\n",
    "# )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c99c13af-d164-4ecc-beb9-66248c70466d",
   "metadata": {},
   "source": [
    "可以看到，返回结果，和上面对应index位置的值是相等的"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e72b05a7-2fe2-4626-a055-57c819fe9a05",
   "metadata": {},
   "source": [
    "### GTE Reranker"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e7703be7-b202-4c0e-b872-4a65a006912e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T01:59:58.044265Z",
     "iopub.status.busy": "2024-09-02T01:59:58.044050Z",
     "iopub.status.idle": "2024-09-02T02:00:04.786838Z",
     "shell.execute_reply": "2024-09-02T02:00:04.786457Z",
     "shell.execute_reply.started": "2024-09-02T01:59:58.044248Z"
    }
   },
   "source": [
    "官方样例，https://huggingface.co/Alibaba-NLP/gte-multilingual-reranker-base：\n",
    "\n",
    "```python\n",
    "import torch\n",
    "from transformers import AutoModelForSequenceClassification, AutoTokenizer\n",
    "\n",
    "model_name_or_path = \"Alibaba-NLP/gte-multilingual-reranker-base\"\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_name_or_path)\n",
    "model = AutoModelForSequenceClassification.from_pretrained(\n",
    "    model_name_or_path, trust_remote_code=True,\n",
    "    torch_dtype=torch.float16\n",
    ")\n",
    "model.eval()\n",
    "\n",
    "pairs = [[\"中国的首都在哪儿\"，\"北京\"], [\"what is the capital of China?\", \"北京\"], [\"how to implement quick sort in python?\",\"Introduction of quick sort\"]]\n",
    "with torch.no_grad():\n",
    "    inputs = tokenizer(pairs, padding=True, truncation=True, return_tensors='pt', max_length=512)\n",
    "    scores = model(**inputs, return_dict=True).logits.view(-1, ).float()\n",
    "    print(scores)\n",
    "\n",
    "# tensor([1.2315, 0.5923, 0.3041])\n",
    "```\n",
    "\n",
    "Langchain中有一个HuggingFaceCrossEncoder也可以CrosssEncoder，但GTE官方样例没有提到说可以使用Langchain的HuggingFaceCrossEncoder计算。\n",
    "\n",
    "```python\n",
    "from langchain_community.cross_encoders import HuggingFaceCrossEncoder\n",
    "\n",
    "reranker = HuggingFaceCrossEncoder(\n",
    "    model_name=model_name_or_path,\n",
    "    model_kwargs={'trust_remote_code': True}\n",
    ")\n",
    "reranker.score(pairs)\n",
    "# array([0.7744, 0.644 , 0.5757], dtype=float16)\n",
    "```\n",
    "\n",
    "观察两者输出，值并不相同，但排序关系是一致的，通过阅读langchain源代码`CrossEncoder.py`可以发现，`HuggingFaceCrossEncoder`的score结果，只是GTE官方结果，过了一个sigmoid。\n",
    "也就是`torch.sigmoid(torch.tensor([1.2315, 0.5923, 0.3041]))`"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d2f53fa8-074b-4fa4-86ca-e4dc7cd0de09",
   "metadata": {},
   "source": [
    "上面提到，虽然使用HuggingFaceCrossEncoder的score也可以获取排序结果，但为了跟后面的流程保持相同接口，下面还是进行简单封装"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "2a85eaca-f90b-4293-ad02-a721192d4213",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T07:31:33.302288Z",
     "iopub.status.busy": "2024-09-02T07:31:33.302163Z",
     "iopub.status.idle": "2024-09-02T07:31:33.307343Z",
     "shell.execute_reply": "2024-09-02T07:31:33.306924Z",
     "shell.execute_reply.started": "2024-09-02T07:31:33.302276Z"
    }
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "from transformers import AutoModelForSequenceClassification, AutoTokenizer\n",
    "\n",
    "class HuggingfaceReranker:\n",
    "    def __init__(self, model_name, model_kwargs: dict):\n",
    "        self._tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
    "        self._model = AutoModelForSequenceClassification.from_pretrained(\n",
    "            model_name, **model_kwargs\n",
    "        )\n",
    "\n",
    "    def compute_score(self, pairs):\n",
    "        if hasattr(self._model, 'compute_score'):\n",
    "            return self._model.compute_score(pairs)\n",
    "\n",
    "        with torch.no_grad():\n",
    "            inputs = self._tokenizer(pairs, padding=True, truncation=True, return_tensors='pt', max_length=512)\n",
    "            inputs = inputs.to(self._model.device)\n",
    "            scores = self._model(**inputs, return_dict=True).logits.view(-1, ).float()\n",
    "        return scores.cpu().numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "d5d4220b-2bb6-488a-9fb4-360531c67053",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T07:31:33.307866Z",
     "iopub.status.busy": "2024-09-02T07:31:33.307752Z",
     "iopub.status.idle": "2024-09-02T07:31:34.684012Z",
     "shell.execute_reply": "2024-09-02T07:31:34.683610Z",
     "shell.execute_reply.started": "2024-09-02T07:31:33.307854Z"
    }
   },
   "outputs": [],
   "source": [
    "# 此处可以替换为本地绝对路径\n",
    "gte_model_path = 'AlibabaNLP/gte-multilingual-reranker-base'\n",
    "\n",
    "reranker = HuggingfaceReranker(gte_model_path, model_kwargs={'trust_remote_code': True, 'torch_dtype': torch.float16})"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "42ca555e-be2a-4e14-82bf-b4f5b913a783",
   "metadata": {},
   "source": [
    "### Jina Reranker"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c46fe624-a650-4ad7-9162-2a4078a135a3",
   "metadata": {},
   "source": [
    "上面介绍的Cohere Rerank，其实就是Jina的，此处介绍的是开源模型"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "04e2aca5-340f-4677-9a8b-84be525ed337",
   "metadata": {},
   "source": [
    "官方示例，https://huggingface.co/jinaai/jina-reranker-v2-base-multilingual\n",
    "\n",
    "```python\n",
    "from transformers import AutoModelForSequenceClassification\n",
    "\n",
    "model = AutoModelForSequenceClassification.from_pretrained(\n",
    "    'jinaai/jina-reranker-v2-base-multilingual',\n",
    "    torch_dtype=\"auto\",\n",
    "    trust_remote_code=True,\n",
    ")\n",
    "\n",
    "model.to('cuda') # or 'cpu' if no GPU is available\n",
    "model.eval()\n",
    "\n",
    "# Example query and documents\n",
    "query = \"Organic skincare products for sensitive skin\"\n",
    "documents = [\n",
    "    \"Organic skincare for sensitive skin with aloe vera and chamomile.\",\n",
    "    \"New makeup trends focus on bold colors and innovative techniques\",\n",
    "    \"Bio-Hautpflege für empfindliche Haut mit Aloe Vera und Kamille\",\n",
    "    \"Neue Make-up-Trends setzen auf kräftige Farben und innovative Techniken\",\n",
    "    \"Cuidado de la piel orgánico para piel sensible con aloe vera y manzanilla\",\n",
    "    \"Las nuevas tendencias de maquillaje se centran en colores vivos y técnicas innovadoras\",\n",
    "    \"针对敏感肌专门设计的天然有机护肤产品\",\n",
    "    \"新的化妆趋势注重鲜艳的颜色和创新的技巧\",\n",
    "    \"敏感肌のために特別に設計された天然有機スキンケア製品\",\n",
    "    \"新しいメイクのトレンドは鮮やかな色と革新的な技術に焦点を当てています\",\n",
    "]\n",
    "\n",
    "# construct sentence pairs\n",
    "sentence_pairs = [[query, doc] for doc in documents]\n",
    "\n",
    "scores = model.compute_score(sentence_pairs, max_length=1024)\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ec0a1e27-6e64-4a3f-b338-6f3897fe1f28",
   "metadata": {},
   "source": [
    "### 不同Rerank效果对比"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "4938875e-edc0-47b3-992a-0adc8958f9cc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T07:31:34.684932Z",
     "iopub.status.busy": "2024-09-02T07:31:34.684763Z",
     "iopub.status.idle": "2024-09-02T07:31:34.688300Z",
     "shell.execute_reply": "2024-09-02T07:31:34.687846Z",
     "shell.execute_reply.started": "2024-09-02T07:31:34.684919Z"
    }
   },
   "outputs": [],
   "source": [
    "def rerank(reranker, query, retrieved_docs, top_k=5, debug=False):\n",
    "    rerank_scores = reranker.compute_score([[query, doc.page_content] for doc in retrieved_docs])\n",
    "    triads = [(query, doc, score) for doc, score in zip(retrieved_docs, rerank_scores)]\n",
    "    triads = sorted(triads, key=lambda triad: triad[-1], reverse=True)\n",
    "    if debug:\n",
    "        return triads\n",
    "    return [triad[1] for triad in triads][:top_k]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "a52a7737-8041-4802-bc77-a4c4acd6877c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T07:31:34.689109Z",
     "iopub.status.busy": "2024-09-02T07:31:34.688808Z",
     "iopub.status.idle": "2024-09-02T07:31:34.703859Z",
     "shell.execute_reply": "2024-09-02T07:31:34.703443Z",
     "shell.execute_reply.started": "2024-09-02T07:31:34.689095Z"
    }
   },
   "outputs": [],
   "source": [
    "from FlagEmbedding import FlagReranker\n",
    "\n",
    "def get_hit_stat_df(get_retriever_fn, reranker, retirever_multiplier=3):\n",
    "    hit_stat_data = []\n",
    "    \n",
    "    top_k_arr = list(range(1, 9))\n",
    "    for k in tqdm(top_k_arr):\n",
    "        retriever = get_retriever_fn(k * retirever_multiplier)\n",
    "        for idx, row in test_df.iterrows():\n",
    "            question = row['question']\n",
    "            true_uuid = row['uuid']\n",
    "            chunks = retriever.get_relevant_documents(question)[:k * retirever_multiplier]\n",
    "            chunks = rerank(reranker, question, chunks, top_k=k)\n",
    "            retrieved_uuids = [doc.metadata['uuid'] for doc in chunks][:k]\n",
    "    \n",
    "            hit_stat_data.append({\n",
    "                'question': question,\n",
    "                'top_k': k,\n",
    "                'hit': int(true_uuid in retrieved_uuids)\n",
    "            })\n",
    "    hit_stat_df = pd.DataFrame(hit_stat_data)\n",
    "    return hit_stat_df\n",
    "\n",
    "def get_reranker(path):\n",
    "    \"\"\"\n",
    "    屏蔽不同Reranker的差异，封装一个同一个的类获取这些Reranker的示例\n",
    "    \"\"\"\n",
    "    \n",
    "    if path is None:\n",
    "        return LangchainCohereRerank()\n",
    "    if 'bge' in path.lower():\n",
    "        return FlagReranker(path, use_fp16=True)\n",
    "    else:\n",
    "        return HuggingfaceReranker(path, model_kwargs={'trust_remote_code': True, 'torch_dtype': torch.float16, 'device_map': device})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "bc649352-2071-447c-a775-5f1acdde20fc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T07:31:34.704442Z",
     "iopub.status.busy": "2024-09-02T07:31:34.704284Z",
     "iopub.status.idle": "2024-09-02T07:31:34.706825Z",
     "shell.execute_reply": "2024-09-02T07:31:34.706523Z",
     "shell.execute_reply.started": "2024-09-02T07:31:34.704430Z"
    }
   },
   "outputs": [],
   "source": [
    "reranker_dict = {\n",
    "    'cohere-rerank': None,\n",
    "    \n",
    "    # 此处可以替换为本地绝对路径\n",
    "    'bge-reranker-base': 'BAAI/bge-reranker-base',\n",
    "    'bge-reranker-large': 'BAAI/bge-reranker-large',\n",
    "    'bge-reranker-v2-m3': 'BAAI/bge-reranker-v2-m3',\n",
    "    'gte-reranker': 'AlibabaNLP/gte-multilingual-reranker-base',\n",
    "    'jina-reranker': 'jinaai/jina-reranker-v2-base-multilingual'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "e4823c9e-52bb-4098-8845-e064cad99c40",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T07:31:34.707414Z",
     "iopub.status.busy": "2024-09-02T07:31:34.707265Z",
     "iopub.status.idle": "2024-09-02T07:31:34.713692Z",
     "shell.execute_reply": "2024-09-02T07:31:34.713280Z",
     "shell.execute_reply.started": "2024-09-02T07:31:34.707402Z"
    }
   },
   "outputs": [],
   "source": [
    "with_reranker_hit_stat_dfs = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "916a8eba-233e-4ec8-97e7-783935218562",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T07:31:34.714288Z",
     "iopub.status.busy": "2024-09-02T07:31:34.714135Z",
     "iopub.status.idle": "2024-09-02T08:09:14.011725Z",
     "shell.execute_reply": "2024-09-02T08:09:14.011203Z",
     "shell.execute_reply.started": "2024-09-02T07:31:34.714276Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bd9d4a8a18b24b11b035c6cd88102c76",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/8 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6a779b080d134f52b2e00c3b9070b77f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/8 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fc01eb011df441adadce53e7753ed49e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/8 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c194626a464e4b87be5363bbd52d8618",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/8 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "flash_attn is not installed. Using PyTorch native attention implementation.\n",
      "flash_attn is not installed. Using PyTorch native attention implementation.\n",
      "flash_attn is not installed. Using PyTorch native attention implementation.\n",
      "flash_attn is not installed. Using PyTorch native attention implementation.\n",
      "flash_attn is not installed. Using PyTorch native attention implementation.\n",
      "flash_attn is not installed. Using PyTorch native attention implementation.\n",
      "flash_attn is not installed. Using PyTorch native attention implementation.\n",
      "flash_attn is not installed. Using PyTorch native attention implementation.\n",
      "flash_attn is not installed. Using PyTorch native attention implementation.\n",
      "flash_attn is not installed. Using PyTorch native attention implementation.\n",
      "flash_attn is not installed. Using PyTorch native attention implementation.\n",
      "flash_attn is not installed. Using PyTorch native attention implementation.\n",
      "flash_attn is not installed. Using PyTorch native attention implementation.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "310c6adba5994020aa6202274d60b046",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/8 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from FlagEmbedding import FlagReranker\n",
    "\n",
    "# 获取Embedding检索器的方法，返回值依然是一个函数，因为还要设置片段数量\n",
    "get_retriever_fn = build_get_ensemble_retriver_fn(weights=[0.5, 0.5])\n",
    "\n",
    "for reranker_name in reranker_dict:\n",
    "    # API用量用完了，先跳过\n",
    "    if reranker_name == 'cohere-rerank':\n",
    "        continue\n",
    "\n",
    "    model_path = reranker_dict[reranker_name]\n",
    "    reranker = get_reranker(model_path)\n",
    "    hit_stat_df = get_hit_stat_df(get_retriever_fn, reranker)\n",
    "    hit_stat_df['reranker'] = f'w/ {reranker_name}'\n",
    "    with_reranker_hit_stat_dfs.append(hit_stat_df)\n",
    "\n",
    "    # 释放显存\n",
    "    torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "f9b4d0bc-bf49-486c-994b-fb5042aa94a6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T08:09:14.012272Z",
     "iopub.status.busy": "2024-09-02T08:09:14.012145Z",
     "iopub.status.idle": "2024-09-02T08:09:14.015626Z",
     "shell.execute_reply": "2024-09-02T08:09:14.015092Z",
     "shell.execute_reply.started": "2024-09-02T08:09:14.012259Z"
    }
   },
   "outputs": [],
   "source": [
    "with_reranker_hit_stat_df = pd.concat(with_reranker_hit_stat_dfs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "00270548-1b2f-4630-8f60-c523d8aef42b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T09:48:15.380338Z",
     "iopub.status.busy": "2024-09-02T09:48:15.379561Z",
     "iopub.status.idle": "2024-09-02T09:48:15.391776Z",
     "shell.execute_reply": "2024-09-02T09:48:15.391312Z",
     "shell.execute_reply.started": "2024-09-02T09:48:15.380266Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>reranker</th>\n",
       "      <th>top_k</th>\n",
       "      <th>hit_rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>w/ bge-reranker-base</td>\n",
       "      <td>1</td>\n",
       "      <td>0.709677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>w/ bge-reranker-base</td>\n",
       "      <td>2</td>\n",
       "      <td>0.817204</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>w/ bge-reranker-base</td>\n",
       "      <td>3</td>\n",
       "      <td>0.870968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>w/ bge-reranker-base</td>\n",
       "      <td>4</td>\n",
       "      <td>0.924731</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>w/ bge-reranker-base</td>\n",
       "      <td>5</td>\n",
       "      <td>0.924731</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>w/ bge-reranker-base</td>\n",
       "      <td>6</td>\n",
       "      <td>0.935484</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>w/ bge-reranker-base</td>\n",
       "      <td>7</td>\n",
       "      <td>0.946237</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>w/ bge-reranker-base</td>\n",
       "      <td>8</td>\n",
       "      <td>0.956989</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>w/ bge-reranker-large</td>\n",
       "      <td>1</td>\n",
       "      <td>0.741935</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>w/ bge-reranker-large</td>\n",
       "      <td>2</td>\n",
       "      <td>0.860215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>w/ bge-reranker-large</td>\n",
       "      <td>3</td>\n",
       "      <td>0.913978</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>w/ bge-reranker-large</td>\n",
       "      <td>4</td>\n",
       "      <td>0.924731</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>w/ bge-reranker-large</td>\n",
       "      <td>5</td>\n",
       "      <td>0.935484</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>w/ bge-reranker-large</td>\n",
       "      <td>6</td>\n",
       "      <td>0.946237</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>w/ bge-reranker-large</td>\n",
       "      <td>7</td>\n",
       "      <td>0.967742</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>w/ bge-reranker-large</td>\n",
       "      <td>8</td>\n",
       "      <td>0.967742</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>w/ bge-reranker-v2-m3</td>\n",
       "      <td>1</td>\n",
       "      <td>0.784946</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>w/ bge-reranker-v2-m3</td>\n",
       "      <td>2</td>\n",
       "      <td>0.903226</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>w/ bge-reranker-v2-m3</td>\n",
       "      <td>3</td>\n",
       "      <td>0.946237</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>w/ bge-reranker-v2-m3</td>\n",
       "      <td>4</td>\n",
       "      <td>0.956989</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>w/ bge-reranker-v2-m3</td>\n",
       "      <td>5</td>\n",
       "      <td>0.956989</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>w/ bge-reranker-v2-m3</td>\n",
       "      <td>6</td>\n",
       "      <td>0.956989</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>w/ bge-reranker-v2-m3</td>\n",
       "      <td>7</td>\n",
       "      <td>0.967742</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>w/ bge-reranker-v2-m3</td>\n",
       "      <td>8</td>\n",
       "      <td>0.967742</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>w/ gte-reranker</td>\n",
       "      <td>1</td>\n",
       "      <td>0.752688</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>w/ gte-reranker</td>\n",
       "      <td>2</td>\n",
       "      <td>0.849462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>w/ gte-reranker</td>\n",
       "      <td>3</td>\n",
       "      <td>0.924731</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>w/ gte-reranker</td>\n",
       "      <td>4</td>\n",
       "      <td>0.935484</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>w/ gte-reranker</td>\n",
       "      <td>5</td>\n",
       "      <td>0.935484</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>w/ gte-reranker</td>\n",
       "      <td>6</td>\n",
       "      <td>0.956989</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>w/ gte-reranker</td>\n",
       "      <td>7</td>\n",
       "      <td>0.967742</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>w/ gte-reranker</td>\n",
       "      <td>8</td>\n",
       "      <td>0.967742</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>w/ jina-reranker</td>\n",
       "      <td>1</td>\n",
       "      <td>0.774194</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>w/ jina-reranker</td>\n",
       "      <td>2</td>\n",
       "      <td>0.870968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>w/ jina-reranker</td>\n",
       "      <td>3</td>\n",
       "      <td>0.913978</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>w/ jina-reranker</td>\n",
       "      <td>4</td>\n",
       "      <td>0.935484</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>w/ jina-reranker</td>\n",
       "      <td>5</td>\n",
       "      <td>0.946237</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>w/ jina-reranker</td>\n",
       "      <td>6</td>\n",
       "      <td>0.946237</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>w/ jina-reranker</td>\n",
       "      <td>7</td>\n",
       "      <td>0.956989</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>w/ jina-reranker</td>\n",
       "      <td>8</td>\n",
       "      <td>0.956989</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 reranker  top_k  hit_rate\n",
       "0    w/ bge-reranker-base      1  0.709677\n",
       "1    w/ bge-reranker-base      2  0.817204\n",
       "2    w/ bge-reranker-base      3  0.870968\n",
       "3    w/ bge-reranker-base      4  0.924731\n",
       "4    w/ bge-reranker-base      5  0.924731\n",
       "5    w/ bge-reranker-base      6  0.935484\n",
       "6    w/ bge-reranker-base      7  0.946237\n",
       "7    w/ bge-reranker-base      8  0.956989\n",
       "8   w/ bge-reranker-large      1  0.741935\n",
       "9   w/ bge-reranker-large      2  0.860215\n",
       "10  w/ bge-reranker-large      3  0.913978\n",
       "11  w/ bge-reranker-large      4  0.924731\n",
       "12  w/ bge-reranker-large      5  0.935484\n",
       "13  w/ bge-reranker-large      6  0.946237\n",
       "14  w/ bge-reranker-large      7  0.967742\n",
       "15  w/ bge-reranker-large      8  0.967742\n",
       "16  w/ bge-reranker-v2-m3      1  0.784946\n",
       "17  w/ bge-reranker-v2-m3      2  0.903226\n",
       "18  w/ bge-reranker-v2-m3      3  0.946237\n",
       "19  w/ bge-reranker-v2-m3      4  0.956989\n",
       "20  w/ bge-reranker-v2-m3      5  0.956989\n",
       "21  w/ bge-reranker-v2-m3      6  0.956989\n",
       "22  w/ bge-reranker-v2-m3      7  0.967742\n",
       "23  w/ bge-reranker-v2-m3      8  0.967742\n",
       "24        w/ gte-reranker      1  0.752688\n",
       "25        w/ gte-reranker      2  0.849462\n",
       "26        w/ gte-reranker      3  0.924731\n",
       "27        w/ gte-reranker      4  0.935484\n",
       "28        w/ gte-reranker      5  0.935484\n",
       "29        w/ gte-reranker      6  0.956989\n",
       "30        w/ gte-reranker      7  0.967742\n",
       "31        w/ gte-reranker      8  0.967742\n",
       "32       w/ jina-reranker      1  0.774194\n",
       "33       w/ jina-reranker      2  0.870968\n",
       "34       w/ jina-reranker      3  0.913978\n",
       "35       w/ jina-reranker      4  0.935484\n",
       "36       w/ jina-reranker      5  0.946237\n",
       "37       w/ jina-reranker      6  0.946237\n",
       "38       w/ jina-reranker      7  0.956989\n",
       "39       w/ jina-reranker      8  0.956989"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with_reranker_hit_stat_df.groupby(['reranker', 'top_k'])['hit'].mean().reset_index().rename(columns={'hit': 'hit_rate'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "d3f7752d-ba09-437f-b8c2-d9310ceb3473",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T09:49:07.400451Z",
     "iopub.status.busy": "2024-09-02T09:49:07.399674Z",
     "iopub.status.idle": "2024-09-02T09:49:07.406335Z",
     "shell.execute_reply": "2024-09-02T09:49:07.405993Z",
     "shell.execute_reply.started": "2024-09-02T09:49:07.400383Z"
    }
   },
   "outputs": [],
   "source": [
    "retriever_only_hit_stat_df['reranker'] = 'w/o'\n",
    "hit_stat_df = pd.concat([retriever_only_hit_stat_df, with_reranker_hit_stat_df])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "d2726273-f8ba-4bc6-9cae-933d376b3bf4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T09:49:08.040299Z",
     "iopub.status.busy": "2024-09-02T09:49:08.039526Z",
     "iopub.status.idle": "2024-09-02T09:49:08.243695Z",
     "shell.execute_reply": "2024-09-02T09:49:08.243254Z",
     "shell.execute_reply.started": "2024-09-02T09:49:08.040229Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: xlabel='top_k', ylabel='hit'>"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABR8AAAKnCAYAAAAP/zpKAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/TGe4hAAAACXBIWXMAAA9hAAAPYQGoP6dpAABut0lEQVR4nOzdebhWZb0//vdmRkCQEBATwTSGAhFxwgEydTuB5lHLIQWHTioGIiZmoKgMnhJILTE9ip6s9JiKOQu5SdBzHAJMM0zU8FsIDYoCCbg3vz/8+Rx3zLYXW7ev13Wt63rWeu77Xp/17FXau3utu2zNmjVrAgAAAABQw+rVdgEAAAAAQN0kfAQAAAAACiF8BAAAAAAKIXwEAAAAAAohfAQAAAAACiF8BAAAAAAKIXwEAAAAAAohfAQAAAAACtGgtgvY0qqqqvLnP/85LVq0SFlZWW2XAwAAAACfKGvWrMk777yTDh06pF69Dc9t/NSFj3/+85+zww471HYZAAAAAPCJ9vrrr+ezn/3sBtt86sLHFi1aJHn/x9l6661ruRoAAAAA+GR5++23s8MOO5Rytg351IWPHzxqvfXWWwsfAQAAAOAj2pRXGlpwBgAAAAAohPARAAAAAChErYaPv/71rzNgwIB06NAhZWVlueeeezbap6KiIr17907jxo2z8847Z+rUqYXXCQAAAABsvlp95+Py5cuz66675rTTTssxxxyz0favvvpqjjjiiHzzm9/MbbfdlhkzZuSMM87Idtttl/Ly8hqra82aNXnvvfdSWVlZY2NCXdewYcPUr1+/tssAAAAAPkZqNXw87LDDcthhh21y+ylTpqRz58656qqrkiTdunXLrFmzMmnSpBoLH1etWpVFixZlxYoVNTIefFqUlZXls5/9bJo3b17bpQAAAAAfE5+o1a6ffPLJHHTQQdWOlZeXZ9iwYevts3LlyqxcubK0//bbb6+3bVVVVV599dXUr18/HTp0SKNGjTZp1R74tFuzZk3+8pe/5P/9v/+XXXbZxQxIAAAAIMknLHx844030q5du2rH2rVrl7fffjv/+Mc/0rRp07X6jB8/PmPGjNmk8VetWpWqqqrssMMO2WqrrWqkZvi02HbbbfPaa69l9erVwkcAAAAgyadgteuLLrooS5cuLW2vv/76RvvUq1fnfxaocWYJAwAAAP/sEzXzsX379lm8eHG1Y4sXL87WW2+9zlmPSdK4ceM0btx4S5QHAAAAAHzIJ2qK3z777JMZM2ZUO/boo49mn332qaWKAAAAAID1qdXwcdmyZZk7d27mzp2bJHn11Vczd+7cLFy4MMn7j0yfcsoppfbf/OY388orr+Tb3/52fv/73+dHP/pR7rjjjpx33nm1UT4fQ4MGDcrRRx9d22UAAAAAkFoOH5955pnstttu2W233ZIkw4cPz2677ZbRo0cnSRYtWlQKIpOkc+fOuf/++/Poo49m1113zVVXXZUbb7wx5eXltVI/AAAAALB+tfrOx/79+2fNmjXr/X7q1Knr7DNnzpwCq+JfsWrVqjRq1KjwPlvKx7k2AAAAgI+7T9Q7H/n46d+/f4YMGZJhw4alTZs2KS8vz/PPP5/DDjsszZs3T7t27fL1r389f/3rXzfYJ0kmTpyYHj16pFmzZtlhhx1y9tlnZ9myZaV+U6dOTatWrfLwww+nW7duad68eQ499NAsWrRovfU9/fTT2XbbbXPllVcmSd56662cccYZ2XbbbbP11lvnwAMPzLx580rtL7300vTq1Ss33nhjOnfunCZNmtT0TwYAAADwqSF85F92yy23pFGjRpk9e3YmTJiQAw88MLvttlueeeaZPPTQQ1m8eHGOP/749faZMmVKkqRevXq5+uqr88ILL+SWW27Jr371q3z729+u1m/FihX5/ve/n//6r//Kr3/96yxcuDAjRoxYZ12/+tWvcvDBB2fs2LG58MILkyTHHXdclixZkgcffDDPPvtsevfunS9/+cv5+9//Xur38ssv5xe/+EXuuuuu0vtIAQAAANh8tfrYNXXDLrvskv/4j/9IklxxxRXZbbfdMm7cuNL3N910U3bYYYe89NJL+fznP79Wnw8MGzas9LlTp0654oor8s1vfjM/+tGPSsdXr16dKVOm5HOf+1ySZMiQIbnsssvWqunuu+/OKaeckhtvvDFf/epXkySzZs3KU089lSVLlqRx48ZJku9///u55557cuedd+Yb3/hGkvcftb711luz7bbb/qs/DQAAAMCnmvCRf9nuu+9e+jxv3rw89thjad68+VrtFixYUAofP9znA9OnT8/48ePz+9//Pm+//Xbee++9vPvuu1mxYkW22mqrJMlWW21VCh6TZLvttsuSJUuqjfO///u/ue+++3LnnXdWW/l63rx5WbZsWT7zmc9Ua/+Pf/wjCxYsKO3vuOOOgkcAAACAGiB85F/WrFmz0udly5ZlwIABpXcsfth22223zj5J8tprr+XII4/MWWedlbFjx6Z169aZNWtWTj/99KxataoUPjZs2LBav7KysrUWLfrc5z6Xz3zmM7nppptyxBFHlPosW7Ys2223XSoqKtaqrVWrVuutDQAAAICPRvhIjerdu3d+8YtfpFOnTmnQYNNvr2effTZVVVW56qqrUq/e+68iveOOOz5SDW3atMldd92V/v375/jjj88dd9yRhg0bpnfv3nnjjTfSoEGDdOrU6SONDQAAAMCms+AMNeqcc87J3//+95xwwgl5+umns2DBgjz88MMZPHhwKisr19tv5513zurVq3PNNdfklVdeyX/913+VFqL5KNq2bZtf/epX+f3vf58TTjgh7733Xg466KDss88+Ofroo/PII4/ktddeyxNPPJGLL744zzzzzEc+FwAAAADrJnykRnXo0CGzZ89OZWVlDjnkkPTo0SPDhg1Lq1atSjMa12XXXXfNxIkTc+WVV+aLX/xibrvttowfP/5fqqV9+/b51a9+ld/+9rc56aSTUlVVlQceeCAHHHBABg8enM9//vP52te+lj/+8Y9p167dv3QuAAAAANZWtuafX5hXx7399ttp2bJlli5dmq233rrad++++25effXVdO7cOU2aNKmlCuGTyX9+AAAA4NNhQ/naPzPzEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAoRIPaLgAAAAD45Nv3mn1rfMxx/12zscVv9xhRo+MlyZCrBtT4mLVp9wturdHxnv3eKTU6XuJe+6Qx8xEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKIQFZzZRTb9wdWOKeCErAAAAAGxJwsdPqZkzZ+bkk0/O66+/XtulAECdN/OAfjU6Xr9fz6zR8ag73GtsKTV9r1kVlk+ysScfW+NjXvyTO2t8zNqy8LIeNT/oNlvX/JifAJ/Ue81j159S06ZNy4AB/mEMAAAAQHGEj3XAfffdl1atWqWysjJJMnfu3JSVlWXkyJGlNmeccUZOPvnk0v69996bgQMHJklWrlyZb33rW2nbtm2aNGmS/fbbL08//fSWvQgAAAAA6hzhYx2w//7755133smcOXOSvP9IdZs2bVJRUVFqM3PmzPTv3z9J8sILL2TJkiU58MADkyTf/va384tf/CK33HJLfvOb32TnnXdOeXl5/v73v2/pSwEAAACgDhE+1gEtW7ZMr169SmFjRUVFzjvvvMyZMyfLli3Ln/70p7z88svp1+/997JMmzYt5eXladSoUZYvX57rrrsu3/ve93LYYYele/fuueGGG9K0adP853/+Zy1eFQAAAACfdMLHOqJfv36pqKjImjVr8vjjj+eYY45Jt27dMmvWrMycOTMdOnTILrvskuT98PGDR64XLFiQ1atXZ9999y2N1bBhw+y555558cUXa+VaAAAAAKgbrHZdR/Tv3z833XRT5s2bl4YNG6Zr167p379/Kioq8uabb5ZmPS5atChz5szJEUccUcsVAwAAAFDXCR/riA/e+zhp0qRS0Ni/f/9MmDAhb775Zs4///wkyS9/+cv07ds3rVu3TpJ87nOfS6NGjTJ79uzsuOOOSZLVq1fn6aefzrBhw2rlWgC2lJkH9KvR8fr9emaNjgdQF409+dgaH/Pin9xZ42PWJftes+/GG22mcZ+A/ylZ1+613S+4tUbHu7vF92p0vCTJNlvX/JjAJ97H/58YbJJtttkmPXv2zG233ZZrr702SXLAAQfk+OOPz+rVq0uB5IdXuU6SZs2a5ayzzsoFF1yQ1q1bp2PHjvmP//iPrFixIqeffnqtXAsAAAAAdYPwcRM9+71TaruEjerXr1/mzp1bWtW6devW6d69exYvXpwuXbpk+fLlmTFjRiZPnlyt34QJE1JVVZWvf/3reeedd9KnT588/PDD2Wabbbb8RQAAAABQZ1hwpg6ZPHly1qxZk65du5aOzZ07N4sWLUqSPPzww+ncuXN23nnnav2aNGmSq6++On/5y1/y7rvvZtasWdljjz22aO0AAAAA1D3Cx0+R5s2b58orr6ztMgAAAAD4lPDY9afIIYccUtslAAAAAPApInwEAPiEufb8X9b4mEsX31LjY1qBeMM+CSsQF3Gv1TVWIAaADfPYNQAAAABQCOEjAAAAAFAI4SMAAAAAUAjhIwAAAABQCOEjAAAAAFAI4SPrNHXq1LRq1aq2y/jYeO2111JWVpa5c+cWep6KioqUlZXlrbfeKvQ8AAAAAFtCg9ou4JNi4WU9tuj5Oo7+beHnmDlzZk4++eS8/vrrhZ8L4NPg2vN/WeNjDrlqQI2PWZcU8c/nE7bZusbHHOdfuT7xCvl3wQLuNQCAjxszHz/Fpk2blgEDPv7/o3bNmjV57733tli/LWX16tW1XQIAAABAoYSPdcB9992XVq1apbKyMkkyd+7clJWVZeTIkaU2Z5xxRk4++eRq/e69994MHDhwg2Pfc8892WWXXdKkSZOUl5evNUvyiiuuSNu2bdOiRYucccYZGTlyZHr16lWtzY033phu3bqlSZMm6dq1a370ox9t8JwfPHr84IMPZvfdd0/jxo0za9asVFVVZfz48encuXOaNm2aXXfdNXfeeedG+y1YsCBHHXVU2rVrl+bNm2ePPfbI9OnTq52zU6dOGTduXE477bS0aNEiHTt2zI9//OP11lhZWZnTTjstXbt2zcKFC5O8H+b27t07TZo0yU477ZQxY8ZUCz/Lyspy3XXXZeDAgWnWrFnGjh273vFnz56dnj17pkmTJtl7773z/PPPl77729/+lhNOOCHbb799ttpqq/To0SM/+9nPqvW/884706NHjzRt2jSf+cxnctBBB2X58uUf+W8CAAAA8FEIH+uA/fffP++8807mzJmT5P3Hqdu0aZOKiopSm5kzZ6Z///6l/RdeeCFLlizJgQceuN5xV6xYkbFjx+bWW2/N7Nmz89Zbb+VrX/ta6fvbbrstY8eOzZVXXplnn302HTt2zHXXXVdtjNtuuy2jR4/O2LFj8+KLL2bcuHEZNWpUbrnllo1e18iRIzNhwoS8+OKL6dmzZ8aPH59bb701U6ZMyQsvvJDzzjsvJ598cmbOnLnBfsuWLcvhhx+eGTNmZM6cOTn00EMzYMCAUmj4gauuuip9+vTJnDlzcvbZZ+ess87K/Pnz16pr5cqVOe644zJ37tw8/vjj6dixYx5//PGccsopGTp0aH73u9/l+uuvz9SpU9cKGC+99NJ85StfyW9/+9ucdtpp6732Cy64IFdddVWefvrpbLvtthkwYEBppuS7776b3XffPffff3+ef/75fOMb38jXv/71PPXUU0mSRYsW5YQTTshpp52WF198MRUVFTnmmGOyZs2af/lvAgAAALA5vICoDmjZsmV69eqVioqK9OnTJxUVFTnvvPMyZsyYLFu2LEuXLs3LL7+cfv36lfpMmzYt5eXladSo0XrHXb16da699trstddeSZJbbrkl3bp1y1NPPZU999wz11xzTU4//fQMHjw4STJ69Og88sgjWbZsWWmMSy65JFdddVWOOeaYJEnnzp1L4dypp566weu67LLLcvDBByd5P/AbN25cpk+fnn322SdJstNOO2XWrFm5/vrrq13bh/slSevWrbPrrruW9i+//PLcfffduffeezNkyJDS8cMPPzxnn312kuTCCy/MpEmT8thjj6VLly6lNsuWLcsRRxyRlStX5rHHHkvLli2TJGPGjMnIkSNL17TTTjvl8ssvz7e//e1ccsklpf4nnnhi6ffakEsuuaR0Dbfccks++9nP5u67787xxx+f7bffPiNGjCi1Pffcc/Pwww/njjvuyJ577plFixblvffeyzHHHJMdd9wxSdKjR49qY3/UvwkAAADA5jDzsY7o169fKioqsmbNmjz++OM55phj0q1bt8yaNSszZ85Mhw4dsssuu5TaT5s2baOPXDdo0CB77LFHab9r165p1apVXnzxxSTJ/Pnzs+eee1br8+H95cuXZ8GCBTn99NPTvHnz0nbFFVdkwYIFSZLDDjusdPwLX/hCtbH69OlT+vzyyy9nxYoVOfjgg6uNdeutt5bGWle/5P3AcMSIEenWrVtatWqV5s2b58UXX1xr5mPPnj1Ln8vKytK+ffssWbKkWpsTTjghy5cvzyOPPFIKHpNk3rx5ueyyy6rVduaZZ2bRokVZsWLFOmvb0LV/ELAm74enXbp0Kf3ulZWVufzyy9OjR4+0bt06zZs3z8MPP1y6nl133TVf/vKX06NHjxx33HG54YYb8uabb27y3wQAAACgppj5WEf0798/N910U+bNm5eGDRuma9eu6d+/fyoqKvLmm29Wmxm4aNGizJkzJ0cccUShNX0wA/KGG24ozZ78QP369ZO8/+7Bf/zjH0mShg0bVmvTrFmztca6//77s/3221dr17hx4/X2S5IRI0bk0Ucfzfe///3svPPOadq0aY499tisWrWqWrt/Pn9ZWVmqqqqqHTv88MPzk5/8JE8++WS1R9aXLVuWMWPGlGYTfliTJk3WWduGrn1Dvve97+UHP/hBJk+enB49eqRZs2YZNmxY6Xrq16+fRx99NE888UQeeeSRXHPNNbn44ovzv//7v9lqq62SbPhvAgAAAFBThI91xAfvfZw0aVIpaOzfv38mTJiQN998M+eff36p7S9/+cv07ds3rVu33uCY7733Xp555pnSbMb58+fnrbfeSrdu3ZIkXbp0ydNPP51TTjml1Ofpp58ufW7Xrl06dOiQV155JSeddNI6z/HPQeL6dO/ePY0bN87ChQurBambYvbs2Rk0aFC+8pWvJHk/KHzttdc2a4wPnHXWWfniF7+YgQMH5v777y/V0rt378yfPz8777zzJo+1oWv/n//5n3Ts2DFJ8uabb+all14q/e6zZ8/OUUcdVVpAqKqqKi+99FK6d+9e6l9WVpZ99903++67b0aPHp0dd9wxd999d4YPH77Rvwl8XO17zb41Pua4T8A/BseefGyNj3nxT+7ceCP4hNv9gltrdLy7W9TocAAAnxof///VxSbZZptt0rNnz9x222259tprkyQHHHBAjj/++KxevbpaYLcpq1wn78/GO/fcc3P11VenQYMGGTJkSPbee+9SGHnuuefmzDPPTJ8+fdK3b9/cfvvtee6557LTTjuVxhgzZky+9a1vpWXLljn00EOzcuXKPPPMM3nzzTczfPjwTb6+Fi1aZMSIETnvvPNSVVWV/fbbL0uXLs3s2bOz9dZbb/BdhbvsskvuuuuuDBgwIGVlZRk1atRaMxo3x7nnnpvKysoceeSRefDBB7Pffvtl9OjROfLII9OxY8cce+yxqVevXubNm5fnn38+V1xxxWaf47LLLstnPvOZtGvXLhdffHHatGmTo48+unQ9d955Z5544olss802mThxYhYvXlwKH//3f/83M2bMyCGHHJK2bdvmf//3f/OXv/ylFF7W1N8EAAAAYGOEj3VIv379Mnfu3NKq1q1bt0737t2zePHi0qIpy5cvz4wZMzJ58uSNjrfVVlvlwgsvzIknnpg//elP2X///fOf//mfpe9POumkvPLKKxkxYkTefffdHH/88Rk0aFBp1eUkOeOMM7LVVlvle9/7Xi644II0a9YsPXr0yLBhwzb7+i6//PJsu+22GT9+fF555ZW0atUqvXv3zne+850N9ps4cWJOO+209O3bN23atMmFF16Yt99+e7PP/2HDhg1LVVVVDj/88Dz00EMpLy/Pfffdl8suuyxXXnll6dH3M8444yONP2HChAwdOjR/+MMf0qtXr/zyl78sLQ703e9+N6+88krKy8uz1VZb5Rvf+EaOPvroLF26NEmy9dZb59e//nUmT56ct99+OzvuuGOuuuqqHHbYYUlq9m8CAAAAsCHCx03UcfRva7uEjZo8efJaoeLcuXOr7T/88MPp3LnzRh8PHjRoUAYNGpQk63yP4QdGjRqVUaNGlfYPPvjgtcY+8cQTc+KJJ278Av5//fv3z5o1a9Y6XlZWlqFDh2bo0KGb1a9Tp0751a9+Ve3YOeecU21/XY9hf/i369Sp01pjDx8+vNpMwfLy8pSXl6+ztiTrrO2fffgajjzyyHW2ad26de655571jtGtW7c89NBDGzzP5v5NAAAAAD4K4eOnTPPmzXPllVfWyFgrVqzIlClTUl5envr16+dnP/tZpk+fnkcffbRGxgcAAADgk034+ClzyCGH1NhYZWVleeCBBzJ27Ni8++676dKlS37xi1/koIMOqrFzAAAAAPDJJXzkI2vatGmmT59e22VQB808YPNWNN+Yfr+eWaPjAQAAAJumXm0XAAAAAADUTcJHAAAAAKAQwkcAAAAAoBDCRwAAAACgEMJHAAAAAKAQwkcAAAAAoBANarsAPp6mTp2aYcOG5a233qrtUj4WXnvttXTu3Dlz5sxJr169CjtPRUVFvvSlL+XNN99Mq1atCjsPQG3Z/YJba3S8u1vU6HAAAEANEz5uon2v2XeLnm/2ubMLP8fMmTNz8skn5/XXXy/8XAAAAAB8+njs+lNs2rRpGTBgQG2XsVFr1qzJe++9t8X6bSmrV68uZNxVq1YVMi4AAADA5hI+1gH33XdfWrVqlcrKyiTJ3LlzU1ZWlpEjR5banHHGGTn55JOr9bv33nszcODADY59zz33ZJdddkmTJk1SXl6+1izJK664Im3btk2LFi1yxhlnZOTIkWs9lnzjjTemW7duadKkSbp27Zof/ehHGzxnRUVFysrK8uCDD2b33XdP48aNM2vWrFRVVWX8+PHp3LlzmjZtml133TV33nnnRvstWLAgRx11VNq1a5fmzZtnjz32yPTp06uds1OnThk3blxOO+20tGjRIh07dsyPf/zj9dZYWVmZ0047LV27ds3ChQuTvB/m9u7dO02aNMlOO+2UMWPGVAs/y8rKct1112XgwIFp1qxZxo4du8HfIUn+9re/5YQTTsj222+frbbaKj169MjPfvazam369++fIUOGZNiwYWnTpk3Ky8uTvP/3/eBv96UvfSm33HJLysrKqj1KP2vWrOy///5p2rRpdthhh3zrW9/K8uXLN1oXAAAAwKYQPtYB+++/f955553MmTMnyfuPU7dp0yYVFRWlNjNnzkz//v1L+y+88EKWLFmSAw88cL3jrlixImPHjs2tt96a2bNn56233srXvva10ve33XZbxo4dmyuvvDLPPvtsOnbsmOuuu67aGLfddltGjx6dsWPH5sUXX8y4ceMyatSo3HLLLRu9rpEjR2bChAl58cUX07Nnz4wfPz633nprpkyZkhdeeCHnnXdeTj755MycOXOD/ZYtW5bDDz88M2bMyJw5c3LooYdmwIABpdDwA1dddVX69OmTOXPm5Oyzz85ZZ52V+fPnr1XXypUrc9xxx2Xu3Ll5/PHH07Fjxzz++OM55ZRTMnTo0Pzud7/L9ddfn6lTp64VMF566aX5yle+kt/+9rc57bTTNvobvPvuu9l9991z//335/nnn883vvGNfP3rX89TTz1Vrd0tt9ySRo0aZfbs2ZkyZUpeffXVHHvssTn66KMzb968/Pu//3suvvjian0WLFiQQw89NP/2b/+W5557LrfffntmzZqVIUOGbLQuAAAAgE3hnY91QMuWLdOrV69UVFSkT58+qaioyHnnnZcxY8Zk2bJlWbp0aV5++eX069ev1GfatGkpLy9Po0aN1jvu6tWrc+2112avvfZK8n7A1a1btzz11FPZc889c8011+T000/P4MGDkySjR4/OI488kmXLlpXGuOSSS3LVVVflmGOOSZJ07ty5FM6deuqpG7yuyy67LAcffHCS9wO/cePGZfr06dlnn32SJDvttFNmzZqV66+/vtq1fbhfkrRu3Tq77rpraf/yyy/P3XffnXvvvbda0Hb44Yfn7LPPTpJceOGFmTRpUh577LF06dKl1GbZsmU54ogjsnLlyjz22GNp2bJlkmTMmDEZOXJk6Zp22mmnXH755fn2t7+dSy65pNT/xBNPLP1em2L77bfPiBEjSvvnnntuHn744dxxxx3Zc889S8d32WWX/Md//Edpf+TIkenSpUu+973vJUm6dOmS559/vloYOn78+Jx00kkZNmxYaYyrr746/fr1y3XXXZcmTZpscp0AAAAA62LmYx3Rr1+/VFRUZM2aNXn88cdzzDHHpFu3bpk1a1ZmzpyZDh06ZJdddim1nzZt2kYfuW7QoEH22GOP0n7Xrl3TqlWrvPjii0mS+fPnVwvAklTbX758eRYsWJDTTz89zZs3L21XXHFFFixYkCQ57LDDSse/8IUvVBurT58+pc8vv/xyVqxYkYMPPrjaWLfeemtprHX1S94PDEeMGJFu3bqlVatWad68eV588cW1Zj727Nmz9LmsrCzt27fPkiVLqrU54YQTsnz58jzyyCOl4DFJ5s2bl8suu6xabWeeeWYWLVqUFStWrLO2DV37ByorK3P55ZenR48ead26dZo3b56HH354rdp33333avvz58+v9rdLstbfat68eZk6dWq1msvLy1NVVZVXX311nfUAAAAAbA4zH+uI/v3756abbsq8efPSsGHDdO3aNf37909FRUXefPPNajMDFy1alDlz5uSII44otKYPZkDecMMNpdmTH6hfv36S998H+Y9//CNJ0rBhw2ptmjVrttZY999/f7bffvtq7Ro3brzefkkyYsSIPProo/n+97+fnXfeOU2bNs2xxx671sIs/3z+srKyVFVVVTt2+OGH5yc/+UmefPLJao+sL1u2LGPGjCnN8PywD88g/HBtG7r2D3zve9/LD37wg0yePDk9evRIs2bNMmzYsLVq/+dr3hTLli3Lv//7v+db3/rWWt917Nhxs8cDAAAA+GfCxzrig/c+Tpo0qRQ09u/fPxMmTMibb76Z888/v9T2l7/8Zfr27ZvWrVtvcMz33nsvzzzzTGnG3Pz58/PWW2+lW7duSd5/lPfpp5/OKaecUurz9NNPlz63a9cuHTp0yCuvvJKTTjppnef45yBxfbp3757GjRtn4cKF1YLUTTF79uwMGjQoX/nKV5K8H7q99tprmzXGB84666x88YtfzMCBA3P//feXaundu3fmz5+fnXfeeZPH2pRrnz17do466qjSYkFVVVV56aWX0r179w3269KlSx544IFqxz78t/mg5t/97nebVTMAAADA5hA+1hHbbLNNevbsmdtuuy3XXnttkuSAAw7I8ccfn9WrV1cL7DZllevk/dl45557bq6++uo0aNAgQ4YMyd57710KI88999yceeaZ6dOnT/r27Zvbb789zz33XHbaaafSGGPGjMm3vvWttGzZMoceemhWrlyZZ555Jm+++WaGDx++ydfXokWLjBgxIuedd16qqqqy3377ZenSpZk9e3a23nrrDb4/cpdddsldd92VAQMGpKysLKNGjVprRuPmOPfcc1NZWZkjjzwyDz74YPbbb7+MHj06Rx55ZDp27Jhjjz029erVy7x58/L888/niiuu+Mjn2mWXXXLnnXfmiSeeyDbbbJOJEydm8eLFGw0f//3f/z0TJ07MhRdemNNPPz1z587N1KlTk7w/ozN5/72We++9d4YMGZIzzjgjzZo1y+9+97s8+uijpXsIAAAA4F/hnY91SL9+/VJZWVla1bp169bp3r172rdvX1o0Zfny5ZkxY8YmhY9bbbVVLrzwwpx44onZd99907x589x+++2l70866aRcdNFFGTFiRHr37p1XX301gwYNqvaY8RlnnJEbb7wxN998c3r06JF+/fpl6tSp6dy582Zf3+WXX55Ro0Zl/Pjx6datWw499NDcf//9Gx1r4sSJ2WabbdK3b98MGDAg5eXl6d2792af/8OGDRuWMWPG5PDDD88TTzyR8vLy3HfffXnkkUeyxx57ZO+9986kSZOy4447/kvn+e53v5vevXunvLw8/fv3T/v27XP00UdvtF/nzp1z55135q677krPnj1z3XXXlVa7/uAx9Z49e2bmzJl56aWXsv/++2e33XbL6NGj06FDh3+pZgAAAIAPmPm4iWafO7u2S9ioyZMnZ/LkydWOzZ07t9r+ww8/nM6dO2/0UdtBgwZl0KBBSbLO9xh+YNSoURk1alRp/+CDD15r7BNPPDEnnnjixi/g/9e/f/+sWbNmreNlZWUZOnRohg4duln9OnXqlF/96lfVjp1zzjnV9tf1GPaHf7tOnTqtNfbw4cOrzd4sLy9PeXn5OmtLss7a/tk/X0Pr1q1zzz33bLBPRUXFOo8PHDiwWsg8duzYfPazn60WDu+xxx555JFHNloXAAAAwEchfPyUad68ea688soaGWvFihWZMmVKysvLU79+/fzsZz/L9OnT8+ijj9bI+PxrfvSjH2WPPfbIZz7zmcyePTvf+973MmTIkNouCwAAAPgUET5+yhxyyCE1NlZZWVkeeOCBjB07Nu+++266dOmSX/ziFznooINq7Bx8dH/4wx9yxRVX5O9//3s6duyY888/PxdddFFtl1VnjD352Bof8+Kf3FnjY26q3S+4tUbHu7vF92p0vCTJNlvX/JgAAAAUSvjIR9a0adNMnz69tstgPSZNmpRJkybVdhkAAADAp5gFZwAAAACAQggfAQAAAIBCCB8BAAAAgEIIHwEAAACAQlhwBviX/H7x75MkVaursvjtxfnOT76TJf9Y8i+NOc5/NQEAAECdYOYjAAAAAFAI4SPrNHXq1LRq1aq2y/jYeO2111JWVpa5c+fWdikAAAAAnxiebdxEMw/ot0XP1+/XMws/x8yZM3PyySfn9ddfL/xc1IyKiopMmjQpTz31VN5+++3ssssuueCCC3LSSSfV+Ln+9re/5aSTTspzzz2Xv/3tb2nbtm2OOuqojBs3LltvvXWNnw8AAACoe8x8/BSbNm1aBgwYUNtlbNSaNWvy3nvvbbF+W8rq1as3u88TTzyRnj175he/+EWee+65DB48OKecckruu+++Gq+vXr16Oeqoo3LvvffmpZdeytSpUzN9+vR885vfrPFzAQAAAHWT8LEOuO+++9KqVatUVlYmSebOnZuysrKMHDmy1OaMM87IySefXK3fvffem4EDB25w7HvuuSe77LJLmjRpkvLy8rVmSV5xxRVp27ZtWrRokTPOOCMjR45Mr169qrW58cYb061btzRp0iRdu3bNj370ow2es6KiImVlZXnwwQez++67p3Hjxpk1a1aqqqoyfvz4dO7cOU2bNs2uu+6aO++8c6P9FixYkKOOOirt2rVL8+bNs8cee2T69OnVztmpU6eMGzcup512Wlq0aJGOHTvmxz/+8XprrKyszGmnnZauXbtm4cKFSd4Pc3v37p0mTZpkp512ypgxY6qFn2VlZbnuuusycODANGvWLGPHjq025iOPPJImTZrkrbfeqnZ86NChOfDAA5Mk3/nOd3L55Zenb9+++dznPpehQ4fm0EMPzV133bXB37R///4599xzM2zYsGyzzTZp165dbrjhhixfvjyDBw9OixYtsvPOO+fBBx8s9dlmm21y1llnpU+fPtlxxx3z5S9/OWeffXYef/zxDZ4LAAAA4APCxzpg//33zzvvvJM5c+Ykef9x6jZt2qSioqLUZubMmenfv39p/4UXXsiSJUtKoda6rFixImPHjs2tt96a2bNn56233srXvva10ve33XZbxo4dmyuvvDLPPvtsOnbsmOuuu67aGLfddltGjx6dsWPH5sUXX8y4ceMyatSo3HLLLRu9rpEjR2bChAl58cUX07Nnz4wfPz633nprpkyZkhdeeCHnnXdeTj755MycOXOD/ZYtW5bDDz88M2bMyJw5c3LooYdmwIABpdDwA1dddVX69OmTOXPm5Oyzz85ZZ52V+fPnr1XXypUrc9xxx2Xu3Ll5/PHH07Fjxzz++OM55ZRTMnTo0Pzud7/L9ddfn6lTp64VMF566aX5yle+kt/+9rc57bTTqn335S9/Oa1atcovfvGL0rHKysrcfvvtG3yseunSpWnduvVGf89bbrklbdq0yVNPPZVzzz03Z511Vo477rj07ds3v/nNb3LIIYfk61//elasWLHO/n/+859z1113pV+/LfsKAgAAAOCTyzsf64CWLVumV69eqaioSJ8+fVJRUZHzzjsvY8aMybJly7J06dK8/PLL1UKjadOmpby8PI0aNVrvuKtXr861116bvfbaK8n74VW3bt3y1FNPZc8998w111yT008/PYMHD06SjB49Oo888kiWLVtWGuOSSy7JVVddlWOOOSZJ0rlz51I4d+qpp27wui677LIcfPDBSd4P/MaNG5fp06dnn332SZLstNNOmTVrVq6//vpq1/bhfknSunXr7LrrrqX9yy+/PHfffXfuvffeDBkypHT88MMPz9lnn50kufDCCzNp0qQ89thj6dKlS6nNsmXLcsQRR2TlypV57LHH0rJlyyTJmDFjMnLkyNI17bTTTrn88svz7W9/O5dcckmp/4knnlj6vf5Z/fr187WvfS0//elPc/rppydJZsyYkbfeeiv/9m//ts4+d9xxR55++ulcf/31G/wtk2TXXXfNd7/73STJRRddlAkTJqRNmzY588wzk7z/97vuuuvy3HPPZe+99y71O+GEEzJt2rT84x//yIABA3LjjTdu9FwfN9ee/8vaLgEAAAA+lcx8rCP69euXioqKrFmzJo8//niOOeaYdOvWLbNmzcrMmTPToUOH7LLLLqX206ZN2+gj1w0aNMgee+xR2u/atWtatWqVF198MUkyf/787LnnntX6fHh/+fLlWbBgQU4//fQ0b968tF1xxRVZsGBBkuSwww4rHf/CF75Qbaw+ffqUPr/88stZsWJFDj744Gpj3XrrraWx1tUveT8wHDFiRLp165ZWrVqlefPmefHFF9ea+dizZ8/S57KysrRv3z5Lliyp1uaEE07I8uXL88gjj5SCxySZN29eLrvssmq1nXnmmVm0aFG1mYQfrm1d137SSSeloqIif/7zn5O8P3P0iCOOWOfK44899lgGDx6cG264odT/8ccfr1bDbbfdts7rq1+/fj7zmc+kR48epWPt2rVLkrWuedKkSfnNb36TadOmZcGCBRk+fPhatQAAAACsi5mPdUT//v1z0003Zd68eWnYsGG6du2a/v37p6KiIm+++Wa1mYGLFi3KnDlzcsQRRxRa0wczIG+44YbS7MkP1K9fP8n774P8xz/+kSRp2LBhtTbNmjVba6z7778/22+/fbV2jRs3Xm+/JBkxYkQeffTRfP/738/OO++cpk2b5thjj82qVauqtfvn85eVlaWqqqrascMPPzw/+clP8uSTT1Z7ZH3ZsmUZM2ZMaYbnhzVp0mSdta3r2vfYY4987nOfy89//vOcddZZufvuuzN16tS1xpw5c2YGDBiQSZMm5ZRTTikd79OnT+bOnVva/yBQXN/1ffhYWVlZkqx1ze3bt0/79u3TtWvXtG7dOvvvv39GjRqV7bbbbq26AAAAAD5M+FhHfPDex0mTJpWCxv79+2fChAl58803c/7555fa/vKXv0zfvn03+p7A9957L88880xpNuP8+fPz1ltvpVu3bkmSLl265Omnn64Wfj399NOlz+3atUuHDh3yyiuvrPedhf8cJK5P9+7d07hx4yxcuHCz3zk4e/bsDBo0KF/5yleSvB8Uvvbaa5s1xgfOOuusfPGLX8zAgQNz//33l2rp3bt35s+fn5133nmTx1rftZ900km57bbb8tnPfjb16tVbKySuqKjIkUcemSuvvDLf+MY3qn3XtGnTzaphc30QTK5cubKwcwAAAAB1h/Cxjthmm23Ss2fP3Hbbbbn22muTJAcccECOP/74rF69ulpgtymrXCfvz5Q799xzc/XVV6dBgwYZMmRI9t5771IYee655+bMM89Mnz590rdv39x+++157rnnstNOO5XGGDNmTL71rW+lZcuWOfTQQ7Ny5co888wzefPNNzfr8d0WLVpkxIgROe+881JVVZX99tsvS5cuzezZs7P11ltv8P2Ru+yyS+66664MGDAgZWVlGTVq1Fqz+zbHueeem8rKyhx55JF58MEHs99++2X06NE58sgj07Fjxxx77LGpV69e5s2bl+effz5XXHHFZo1/0kkn5dJLL83YsWNz7LHHVpvZ+dhjj+XII4/M0KFD82//9m954403kiSNGjXapEVnNscDDzyQxYsXZ4899kjz5s3zwgsv5IILLsi+++6bTp061ei5AAAAgLrJOx/rkH79+qWysrK0qnXr1q3TvXv3tG/fvrRoyvLlyzNjxoxNCh+32mqrXHjhhTnxxBOz7777pnnz5rn99ttL35900km56KKLMmLEiPTu3TuvvvpqBg0aVO0x4zPOOCM33nhjbr755vTo0SP9+vXL1KlT07lz582+vssvvzyjRo3K+PHj061btxx66KG5//77NzrWxIkTs80226Rv374ZMGBAysvL07t3780+/4cNGzYsY8aMyeGHH54nnngi5eXlue+++/LII49kjz32yN57751JkyZlxx133Oyxd9555+y555557rnn1poxesstt2TFihUZP358tttuu9K2rse9/1VNmzbNDTfckP322y/dunXLeeedl4EDB+a+++6r8XMBAAAAdZOZj5uo369n1nYJGzV58uRMnjy52rEPv/8vSR5++OF07tx5o4/mDho0KIMGDUqSDQZbo0aNyqhRo0r7Bx988Fpjn3jiiTnxxBM3fgH/v/79+2fNmjVrHS8rK8vQoUMzdOjQzerXqVOn/OpXv6p27Jxzzqm2v67HsD/823Xq1GmtsYcPH15t9mZ5eXnKy8vXWVuSdda2Pv/7v/+7zuNTp05d5zsgN6aiomKtY+u65g/X+KUvfSlPPPHEZp8LAAAA4APCx0+Z5s2b58orr6yRsVasWJEpU6akvLw89evXz89+9rNMnz49jz76aI2MDwAAAMAnm/DxU+aQQw6psbHKysrywAMPZOzYsXn33XfTpUuX/OIXv8hBBx1UY+cAAAAA4JNL+MhH1rRp00yfPr22ywAAAADgY8qCMwAAAABAIYSPAAAAAEAhhI8AAAAAQCGEjwAAAABAIYSPAAAAAEAhhI8AAAAAQCGEj7Ael156aXr16lXbZQAAAAB8YjWo7QI+Ka49/5db9HxDrhpQ+DlmzpyZk08+Oa+//vpH6n/ppZfmnnvuydy5c2u2MAAAAADqBDMfP8WmTZuWAQOKDzk/ilWrVm3RfltCZWVlqqqqarsMAAAAgC1G+FgH3HfffWnVqlUqKyuTJHPnzk1ZWVlGjhxZanPGGWfk5JNPrtbv3nvvzcCBA9c77g033JAddtghW221Vb7yla9k4sSJadWqVZJk6tSpGTNmTObNm5eysrKUlZVl6tSpSZK33norZ5xxRrbddttsvfXWOfDAAzNv3rwNXsOgQYNy9NFHZ+zYsenQoUO6dOmSJHn99ddz/PHHp1WrVmndunWOOuqovPbaaxvt91//9V/p06dPWrRokfbt2+fEE0/MkiVLSv0qKipSVlaWGTNmpE+fPtlqq63St2/fzJ8/f701LliwIDvttFOGDBmSNWvWZOXKlRkxYkS23377NGvWLHvttVcqKipK7adOnZpWrVrl3nvvTffu3dO4ceMsXLhwg79D0Vb++YUa3wAAAADWR/hYB+y///555513MmfOnCTvP07dpk2bakHYzJkz079//9L+Cy+8kCVLluTAAw9c55izZ8/ON7/5zQwdOjRz587NwQcfnLFjx5a+/+pXv5rzzz8/X/jCF7Jo0aIsWrQoX/3qV5Mkxx13XJYsWZIHH3wwzz77bHr37p0vf/nL+fvf/77B65gxY0bmz5+fRx99NPfdd19Wr16d8vLytGjRIo8//nhmz56d5s2b59BDD602w/Gf+yXJ6tWrc/nll2fevHm555578tprr2XQoEFrnfPiiy/OVVddlWeeeSYNGjTIaaedts7annvuuey333458cQTc+2116asrCxDhgzJk08+mZ///Od57rnnctxxx+XQQw/NH/7wh1K/FStW5Morr8yNN96YF154IW3btt3gbwAAAABQl3jnYx3QsmXL9OrVKxUVFenTp08qKipy3nnnZcyYMVm2bFmWLl2al19+Of369Sv1mTZtWsrLy9OoUaN1jnnNNdfksMMOy4gRI5Ikn//85/PEE0+Uwr2mTZumefPmadCgQdq3b1/qN2vWrDz11FNZsmRJGjdunCT5/ve/n3vuuSd33nlnvvGNb6z3Opo1a5Ybb7yxVNNPfvKTVFVV5cYbb0xZWVmS5Oabb06rVq1SUVGRQw45ZJ39klQLEXfaaadcffXV2WOPPbJs2bI0b9689N3YsWNLv8vIkSNzxBFH5N13302TJk1KbZ544okceeSRufjii3P++ecnSRYuXJibb745CxcuTIcOHZIkI0aMyEMPPZSbb74548aNS/J+CPqjH/0ou+6663qvGwAAAKCuMvOxjujXr18qKiqyZs2aPP744znmmGPSrVu3zJo1KzNnzkyHDh2yyy67lNpPmzZtg49cz58/P3vuuWe1Y/+8vy7z5s3LsmXL8pnPfCbNmzcvba+++moWLFiQhQsXVjv+QUiXJD169KgWIM6bNy8vv/xyWrRoUWrfunXrvPvuu1mwYMF6+yXJs88+mwEDBqRjx45p0aJFKWD858eee/bsWfq83XbbJUm1x7MXLlyYgw8+OKNHjy4Fj0ny29/+NpWVlfn85z9f7XpmzpxZrbZGjRpVOwcAAADAp4mZj3VE//79c9NNN2XevHlp2LBhunbtmv79+6eioiJvvvlmtVmPixYtypw5c3LEEUfUeB3Lli3LdtttV+2R7w+0atUqrVq1qrY6duvWrUufmzVrttZYu+++e2677ba1xtp2223X22/58uUpLy9PeXl5brvttmy77bZZuHBhysvL11qQpmHDhqXPH8yu/PCiMNtuu206dOiQn/3sZznttNOy9dZbl2qrX79+nn322dSvX7/amB+eWdm0adPSuAAAAACfNsLHOuKD9z5OmjSpFDT2798/EyZMyJtvvllt1t4vf/nL9O3bt1rw98+6dOmSp59+utqxf95v1KhRaZGbD/Tu3TtvvPFGGjRokE6dOq1z7J133nmTrql37965/fbb07Zt21Lotyl+//vf529/+1smTJiQHXbYIUnyzDPPbHL/D2vatGnuu+++HH744SkvL88jjzySFi1aZLfddktlZWWWLFmS/fff/yONDQAAAFDXeey6jthmm23Ss2fP3HbbbaWFZQ444ID85je/yUsvvVRt5uPGVrlOknPPPTcPPPBAJk6cmD/84Q+5/vrr8+CDD1abxdepU6e8+uqrmTt3bv76179m5cqVOeigg7LPPvvk6KOPziOPPJLXXnstTzzxRC6++OLNDgBPOumktGnTJkcddVQef/zxvPrqq6moqMi3vvWt/L//9//W269jx45p1KhRrrnmmrzyyiu59957c/nll2/WuT+sWbNmuf/++9OgQYMcdthhWbZsWT7/+c/npJNOyimnnJK77rorr776ap566qmMHz8+999//0c+FwAAAEBdUuvh4w9/+MN06tQpTZo0yV577ZWnnnpqg+0nT56cLl26pGnTptlhhx1y3nnn5d13391C1X689evXL5WVlaXwsXXr1unevXvat2+fLl26JHn/keQZM2ZsNHzcd999M2XKlEycODG77rprHnrooZx33nnVFmL5t3/7txx66KH50pe+lG233TY/+9nPUlZWlgceeCAHHHBABg8enM9//vP52te+lj/+8Y9p167dZl3PVlttlV//+tfp2LFj6R2Wp59+et59990NzoTcdtttM3Xq1Pz3f/93unfvngkTJuT73//+Zp37nzVv3jwPPvhg1qxZkyOOOCLLly/PzTffnFNOOSXnn39+unTpkqOPPjpPP/10Onbs+C+dCwAAAKCuqNXHrm+//fYMHz48U6ZMyV577ZXJkyenvLw88+fPT9u2bddq/9Of/jQjR47MTTfdlL59++all17KoEGDUlZWlokTJxZa65CrBhQ6fk2YPHlyJk+eXO3Yh9+vmCQPP/xwOnfuvEmPPp955pk588wzq+1/uF/jxo1z5513rtWvRYsWufrqq3P11Vdvcu1Tp05d5/H27dvnlltu2ex+J5xwQk444YRqx9asWVP63L9//2r7SdKrV69qxy699NJceumlpf3mzZtn9uzZ1fqMGTMmY8aMWWcNgwYNyqBBg9ZbOwAAAEBdV6szHydOnJgzzzwzgwcPTvfu3TNlypRstdVWuemmm9bZ/oknnsi+++6bE088MZ06dcohhxySE044YaOzJfk/zZs3z5VXXrlJbb///e+XVpy+5pprcsstt+TUU08tuEIAAAAA6opaCx9XrVqVZ599NgcddND/FVOvXg466KA8+eST6+zTt2/fPPvss6Ww8ZVXXskDDzyQww8/fL3nWblyZd5+++1q26fZIYcckgEDNm0W51NPPZWDDz44PXr0yJQpU3L11VfnjDPOKLhCAAAAAOqKWnvs+q9//WsqKyvXeg9gu3bt8vvf/36dfU488cT89a9/zX777Zc1a9bkvffeyze/+c185zvfWe95xo8fv97HYtmwO+64o7ZLAAAAAOATrNYXnNkcFRUVGTduXH70ox/lN7/5Te66667cf//9G1zJ+KKLLsrSpUtL2+uvv74FKwYAAACAT69am/nYpk2b1K9fP4sXL652fPHixWnfvv06+4waNSpf//rXS4/+9ujRI8uXL883vvGNXHzxxalXb+0stXHjxmncuHHNXwAAAAAAsEG1NvOxUaNG2X333TNjxozSsaqqqsyYMSP77LPPOvusWLFirYCxfv36SbLWysUAAAAAQO2qtZmPSTJ8+PCceuqp6dOnT/bcc89Mnjw5y5cvz+DBg5Mkp5xySrbffvuMHz8+STJgwIBMnDgxu+22W/baa6+8/PLLGTVqVAYMGFAKIQEAAACAj4daDR+/+tWv5i9/+UtGjx6dN954I7169cpDDz1UWoRm4cKF1WY6fve7301ZWVm++93v5k9/+lO23XbbDBgwIGPHjq2tSwAAAAAA1qNWw8ckGTJkSIYMGbLO7yoqKqrtN2jQIJdcckkuueSSLVAZAAAAAPCv+EStds2WU1FRkbKysrz11ltJkqlTp6ZVq1a1WlNtee2111JWVpa5c+fWdikAAAAAnyi1PvPxk2Lsycdu0fNd/JM7Cz/HzJkzc/LJJ+f1119f67u+fftm0aJFadmyZZL3H5E//PDDC68JAAAAgLpD+PgpNm3atAwYMGCd3zVq1Cjt27cv7Tdt2jTv/fGPeefvf6+x87fo2nWjbdasWZPKyso0aLB5t+pH7belrF69Og0bNqztMgAAAAAK5bHrOuC+++5Lq1atUllZmSSZO3duysrKMnLkyFKbM844IyeffHK1fvfee28GDhy4zjHX9dj1DnvuWfp+3LXXZt+vfCU/mzYtX/zyl/PZPfbIoOHD887y5aU2jz7+eA456aTssOee2XHvvXPcN7+ZVxYu3OC1fHDeBx98MLvvvnsaN26cWbNmpaqqKuPHj0/nzp3TtGnT7Lrrrrnzzjs32m/BggU56qij0q5duzRv3jx77LFHpk+fXu2cnTp1yrhx43LaaaelRYsW6dixY3784x+vt8bKysqcdtpp6dq1axb+/9czbdq09O7dO02aNMlOO+2UMWPG5L333iv1KSsry3XXXZeBAwemWbNmFkkCAAAAPhWEj3XA/vvvn3feeSdz5sxJ8v7j1G3atKm2YM/MmTPTv3//0v4LL7yQJUuW5MADD/zI53114cLcP2NG7rjuutxx3XWZ/fTTmXTDDaXvV/zjHxly6qmp+O//zi9vvjn16tXLSeeem6qqqo2OPXLkyEyYMCEvvvhievbsmfHjx+fWW2/NlClT8sILL+S8887LySefnJkzZ26w37Jly3L44YdnxowZmTNnTg499NAMGDCgFBp+4KqrrkqfPn0yZ86cnH322TnrrLMyf/78tepauXJljjvuuMydOzePP/54OnbsmMcffzynnHJKhg4dmt/97ne5/vrrM3Xq1LUCxksvvTRf+cpX8tvf/jannXba5vzUAAAAAJ9IH89nUtksLVu2TK9evVJRUZE+ffqkoqIi5513XsaMGZNly5Zl6dKlefnll9OvX79Sn2nTpqW8vDyNGjX6yOetWrMm140fnxbNmiVJvjZwYCqefDKjhw1Lkhx1yCHV2v9o7Nh07ts3v3/55XT//Oc3OPZll12Wgw8+OMn7gd+4ceMyffr07LPPPkmSnXbaKbNmzcr1119f7bo+3C9JWrdunV133bW0f/nll+fuu+/OvffeW22V9cMPPzxnn312kuTCCy/MpEmT8thjj6VLly6lNsuWLcsRRxyRlStX5rHHHiu9D3PMmDEZOXJkTj311FJtl19+eb797W9XW5n9xBNPzODBgzd43QAAAAB1iZmPdUS/fv1SUVGRNWvW5PHHH88xxxyTbt26ZdasWZk5c2Y6dOiQXXbZpdR+2rRp633kelN17NChFDwmSbttt81fP/ROyJdfey2Dzz8/PQ8+ONv36ZMvHnRQkuT1RYuSJIcddliaN2+e5s2b5wtf+EK1sfv06fN/47z8clasWJGDDz641L558+a59dZbs2DBgvX2S94PDEeMGJFu3bqlVatWad68eV588cW1Zj727Nmz9LmsrCzt27fPkiVLqrU54YQTsnz58jzyyCOl4DFJ5s2bl8suu6xabWeeeWYWLVqUFStWrLc2AAAAgLrOzMc6on///rnpppsyb968NGzYMF27dk3//v1TUVGRN998s9rswEWLFmXOnDk54ogj/qVz/vOCKWVlZdUeqf7q2Wdnhw4dcvVll2W7tm1TVVWVvQYOzOrVq5MkN954Y/7xj3+sc6xmHwo1ly1bliS5//77s/3221dr17hx4/X2S5IRI0bk0Ucfzfe///3svPPOadq0aY499tisWrVqs64leX925E9+8pM8+eST1R5XX7ZsWcaMGZNjjjkm/6xJkybrrQ0AAACgrhM+1hEfvPdx0qRJpaCxf//+mTBhQt58882cf/75pba//OUv07dv37Ru3bqwev725pv5w6uv5prLLkvf/3/G35PPPlutzT8HievTvXv3NG7cOAsXLqwWoq7LG6+9mn9svXVpf+Zjj+XfBg7M3ru+P7Nx+fLlefWVV7LHbrtl0Svvz5qsfO+9vP23v5b2k2T1qlV5582/v3+sXv0kyVlnnZUvfvGLGThwYO6///5SLb179878+fOz8847b9L1AAAAAHxaCB/riG222SY9e/bMbbfdlmuvvTZJcsABB+T444/P6tWrq4V2G1rlusbqadkyrVu1ys133JF2226b/7doUS6ZOPEjjdWiRYuMGDEi5513XqqqqrLffvtl6dKlmT17drbeeuvSuxbXpXOnTnngkYdz8JcPTFlZWf5j0qRUrdn4gjfrc+6556aysjJHHnlkHnzwwey3334ZPXp0jjzyyHTs2DHHHnts6tWrl3nz5uX555/PFVdc8ZHPBQAAAPBJ552PdUi/fv1SWVlZWtW6devW6d69e9q3b19aOGX58uWZMWNG4eFjvXr1cvNVV2Xu736XvQcOzEUTJuSKESM+8niXX355Ro0alfHjx6dbt2459NBDc//996dz584b7Hfpxd9Jq61bZuBxx+fUM7+R/vvvnx7/9H7JzTVs2LCMGTMmhx9+eJ544omUl5fnvvvuyyOPPJI99tgje++9dyZNmpQdd9zxXzoPAAAAwCedmY+b6OKf3FnbJWzU5MmTM3ny5GrH5s6dW23/4YcfTufOnTf6iPDKlStTVlaWrbbaKkkyaNCg/Nvee5e+/86QIfnOh1aLTpJzTj0153xoFuKX+vbN0/fdV63N2y++uMHz9u/fP2vWrFnreFlZWYYOHZqhQ4dusN+HH51Okh0++9n8920/qXZs8Ne/Xm3/qV/PXGu86ff9svS5U6dOa9U0fPjwDB8+vLRfXl6e8vLy9VxV1nlNAAAAAHWdmY+fMs2bN8+VV165wTaLFy/OtGnTsssuu6RRo0ZbqDIAAAAA6hozHz9lDjnkkI22Ofzww/POO+/kRz/60RaoCAAAAIC6SvjIWp79p1WpAQAAAOCj8Ng1AAAAAFAI4SMAAAAAUAjh4zpUVVXVdgnwibQma6zsDQAAAJR45+OHNGrUKPXq1cuf//znbLvttmnUqFHKyspqu6yPjVU1HMq+++67NTpekqyurKzxMYuos7aseq/mg/Wqqvf/M7Jy2cosX708S1ctrfFzAAAAAJ9MwscPqVevXjp37pxFixblz3/+c22X87Hz7uLFNTpekwKC3aV//UuNj7n8vZoPNGvLe28tqfEx/1qvXtZkTZavXp6f/+HnWVW1qsbPAQAAAHwyCR//SaNGjdKxY8e89957qSxgFt0n2VOjRtfoeN1+8l81Ol6STPnRpBof85vfu7rGx6wtf/7h0Bofc3TLZlmzZk2WrloqeAQAAACqET6uQ1lZWRo2bJiGDRvWdikfK2uW1OysuRsvfrRGx0uS5W/+vcbHbNKkSY2Pual2v+DWGh3v7haLanS8JFnSaOsaHxMAAACoGyw4AwAAAAAUQvgIAAAAABRC+AgAAAAAFEL4CAAAAAAUQvgIAAAAABRC+AgAAAAAFKJBbRdAMfa9Zt8aH3Oc2wUAAACAzWDmIwAAAABQCOEjAAAAAFAI4SMAAAAAUAjhIwAAAABQCOEjAAAAAFAI4SMAAAAAUAjhIwAAAABQCOEjAAAAAFAI4SMAAAAAUAjhIwAAAABQCOEjAAAAAFAI4SMAAAAAUAjhIwAAAABQCOEjAAAAAFAI4SMAAAAAUAjhIwAAAABQCOEjAAAAAFAI4SMAAAAAUIgGtV0AycLLetT8oNtsXfNjAgAAAMBmMPMRAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKESth48//OEP06lTpzRp0iR77bVXnnrqqQ22f+utt3LOOedku+22S+PGjfP5z38+DzzwwBaqFgAAAADYVA1q8+S33357hg8fnilTpmSvvfbK5MmTU15envnz56dt27ZrtV+1alUOPvjgtG3bNnfeeWe23377/PGPf0yrVq22fPEAAAAAwAbVavg4ceLEnHnmmRk8eHCSZMqUKbn//vtz0003ZeTIkWu1v+mmm/L3v/89TzzxRBo2bJgk6dSp05YsGQAAAADYRLX22PWqVavy7LPP5qCDDvq/YurVy0EHHZQnn3xynX3uvffe7LPPPjnnnHPSrl27fPGLX8y4ceNSWVm53vOsXLkyb7/9drUNAAAAACherYWPf/3rX1NZWZl27dpVO96uXbu88cYb6+zzyiuv5M4770xlZWUeeOCBjBo1KldddVWuuOKK9Z5n/PjxadmyZWnbYYcdavQ6AAAAAIB1q/UFZzZHVVVV2rZtmx//+MfZfffd89WvfjUXX3xxpkyZst4+F110UZYuXVraXn/99S1YMQAAAAB8etXaOx/btGmT+vXrZ/HixdWOL168OO3bt19nn+222y4NGzZM/fr1S8e6deuWN954I6tWrUqjRo3W6tO4ceM0bty4ZosHAAAAADaq1mY+NmrUKLvvvntmzJhROlZVVZUZM2Zkn332WWeffffdNy+//HKqqqpKx1566aVst9126wweAQAAAIDaU6uPXQ8fPjw33HBDbrnllrz44os566yzsnz58tLq16ecckouuuiiUvuzzjorf//73zN06NC89NJLuf/++zNu3Licc845tXUJAAAAAMB61Npj10ny1a9+NX/5y18yevTovPHGG+nVq1ceeuih0iI0CxcuTL16/5eP7rDDDnn44Ydz3nnnpWfPntl+++0zdOjQXHjhhbV1CQAAAADAetRq+JgkQ4YMyZAhQ9b5XUVFxVrH9tlnn/zP//xPwVUBAAAAAP+qT9Rq1wAAAADAJ4fwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAAChEg9ou4JNo9wturdHx7m5Ro8MBAAAAwMeCmY8AAAAAQCGEjwAAAABAIYSPAAAAAEAhhI8AAAAAQCGEjwAAAABAIYSPAAAAAEAhhI8AAAAAQCGEjwAAAABAIYSPAAAAAEAhhI8AAAAAQCGEjwAAAABAIYSPAAAAAEAhhI8AAAAAQCGEjwAAAABAIYSPAAAAAEAhhI8AAAAAQCGEjwAAAABAIYSPAAAAAEAhPlL4eOCBB+att95a6/jbb7+dAw888F+tCQAAAACoAz5S+FhRUZFVq1atdfzdd9/N448//i8XBQAAAAB88jXYnMbPPfdc6fPvfve7vPHGG6X9ysrKPPTQQ9l+++1rrjoAAAAA4BNrs8LHXr16paysLGVlZet8vLpp06a55ppraqw4AAAAAOCTa7PCx1dffTVr1qzJTjvtlKeeeirbbrtt6btGjRqlbdu2qV+/fo0XCQAAAAB88mxW+LjjjjsmSaqqqgopBgAAAACoOzY5fLz33ntz2GGHpWHDhrn33ns32HbgwIH/cmEAAAAAwCfbJoePRx99dN544420bds2Rx999HrblZWVpbKysiZqAwAAAAA+wTY5fPzwo9YeuwYAAAAANmaz3vn4YTNmzMiMGTOyZMmSamFkWVlZ/vM//7NGigMAAAAAPrk+Uvg4ZsyYXHbZZenTp0+22267lJWV1XRdAAAAAMAn3EcKH6dMmZKpU6fm61//ek3XAwAAAADUEfU+SqdVq1alb9++NV0LAAAAAFCHfKTw8YwzzshPf/rTmq4FAAAAAKhDNvmx6+HDh5c+V1VV5cc//nGmT5+enj17pmHDhtXaTpw4seYqBAAAAAA+kTY5fJwzZ061/V69eiVJnn/++WrHLT4DAAAAACSbET4+9thjRdYBAAAAANQxH+mdjwAAAAAAGyN8BAAAAAAKIXwEAAAAAAohfAQAAAAACiF8BAAAAAAKIXwEAAAAAAohfAQAAAAACiF8BAAAAAAKIXwEAAAAAAohfAQAAAAACiF8BAAAAAAKIXwEAAAAAAohfAQAAAAACiF8BAAAAAAKIXwEAAAAAAohfAQAAAAACiF8BAAAAAAKIXwEAAAAAAohfAQAAAAACiF8BAAAAAAKIXwEAAAAAAohfAQAAAAACiF8BAAAAAAKIXwEAAAAAAohfAQAAAAACiF8BAAAAAAKIXwEAAAAAAohfAQAAAAACiF8BAAAAAAKIXwEAAAAAAohfAQAAAAACiF8BAAAAAAKIXwEAAAAAAohfAQAAAAACiF8BAAAAAAKIXwEAAAAAAohfAQAAAAACiF8BAAAAAAKIXwEAAAAAAohfAQAAAAACiF8BAAAAAAKIXwEAAAAAAohfAQAAAAACiF8BAAAAAAKIXwEAAAAAAohfAQAAAAACiF8BAAAAAAKIXwEAAAAAAohfAQAAAAACiF8BAAAAAAK8bEIH3/4wx+mU6dOadKkSfbaa6889dRTm9Tv5z//ecrKynL00UcXWyAAAAAAsNlqPXy8/fbbM3z48FxyySX5zW9+k1133TXl5eVZsmTJBvu99tprGTFiRPbff/8tVCkAAAAAsDlqPXycOHFizjzzzAwePDjdu3fPlClTstVWW+Wmm25ab5/KysqcdNJJGTNmTHbaaactWC0AAAAAsKlqNXxctWpVnn322Rx00EGlY/Xq1ctBBx2UJ598cr39LrvssrRt2zann376Rs+xcuXKvP3229U2AAAAAKB4tRo+/vWvf01lZWXatWtX7Xi7du3yxhtvrLPPrFmz8p//+Z+54YYbNukc48ePT8uWLUvbDjvs8C/XDQAAAABsXK0/dr053nnnnXz961/PDTfckDZt2mxSn4suuihLly4tba+//nrBVQIAAAAASdKgNk/epk2b1K9fP4sXL652fPHixWnfvv1a7RcsWJDXXnstAwYMKB2rqqpKkjRo0CDz58/P5z73uWp9GjdunMaNGxdQPQAAAACwIbU687FRo0bZfffdM2PGjNKxqqqqzJgxI/vss89a7bt27Zrf/va3mTt3bmkbOHBgvvSlL2Xu3LkeqQYAAACAj5FanfmYJMOHD8+pp56aPn36ZM8998zkyZOzfPnyDB48OElyyimnZPvtt8/48ePTpEmTfPGLX6zWv1WrVkmy1nEAAAAAoHbVevj41a9+NX/5y18yevTovPHGG+nVq1ceeuih0iI0CxcuTL16n6hXUwIAAAAA+RiEj0kyZMiQDBkyZJ3fVVRUbLDv1KlTa74gAAAAAOBfZkohAAAAAFAI4SMAAAAAUAjhIwAAAABQCOEjAAAAAFAI4SMAAAAAUAjhIwAAAABQCOEjAAAAAFAI4SMAAAAAUAjhIwAAAABQCOEjAAAAAFAI4SMAAAAAUAjhIwAAAABQCOEjAAAAAFAI4SMAAAAAUAjhIwAAAABQCOEjAAAAAFAI4SMAAAAAUAjhIwAAAABQCOEjAAAAAFAI4SMAAAAAUAjhIwAAAABQCOEjAAAAAFAI4SMAAAAAUAjhIwAAAABQCOEjAAAAAFAI4SMAAAAAUAjhIwAAAABQCOEjAAAAAFAI4SMAAAAAUAjhIwAAAABQCOEjAAAAAFAI4SMAAAAAUAjhIwAAAABQCOEjAAAAAFAI4SMAAAAAUAjhIwAAAABQCOEjAAAAAFAI4SMAAAAAUAjhIwAAAABQCOEjAAAAAFAI4SMAAAAAUAjhIwAAAABQCOEjAAAAAFAI4SMAAAAAUAjhIwAAAABQCOEjAAAAAFAI4SMAAAAAUAjhIwAAAABQCOEjAAAAAFAI4SMAAAAAUAjhIwAAAABQCOEjAAAAAFAI4SMAAAAAUAjhIwAAAABQCOEjAAAAAFAI4SMAAAAAUAjhIwAAAABQCOEjAAAAAFAI4SMAAAAAUAjhIwAAAABQCOEjAAAAAFAI4SMAAAAAUAjhIwAAAABQCOEjAAAAAFAI4SMAAAAAUAjhIwAAAABQCOEjAAAAAFAI4SMAAAAAUAjhIwAAAABQCOEjAAAAAFAI4SMAAAAAUAjhIwAAAABQCOEjAAAAAFAI4SMAAAAAUAjhIwAAAABQCOEjAAAAAFAI4SMAAAAAUAjhIwAAAABQCOEjAAAAAFAI4SMAAAAAUAjhIwAAAABQCOEjAAAAAFAI4SMAAAAAUAjhIwAAAABQCOEjAAAAAFAI4SMAAAAAUAjhIwAAAABQCOEjAAAAAFAI4SMAAAAAUAjhIwAAAABQCOEjAAAAAFAI4SMAAAAAUAjhIwAAAABQCOEjAAAAAFAI4SMAAAAAUAjhIwAAAABQCOEjAAAAAFAI4SMAAAAAUAjhIwAAAABQCOEjAAAAAFAI4SMAAAAAUAjhIwAAAABQCOEjAAAAAFAI4SMAAAAAUAjhIwAAAABQCOEjAAAAAFAI4SMAAAAAUAjhIwAAAABQCOEjAAAAAFAI4SMAAAAAUIiPRfj4wx/+MJ06dUqTJk2y11575amnnlpv2xtuuCH7779/ttlmm2yzzTY56KCDNtgeAAAAAKgdtR4+3n777Rk+fHguueSS/OY3v8muu+6a8vLyLFmyZJ3tKyoqcsIJJ+Sxxx7Lk08+mR122CGHHHJI/vSnP23hygEAAACADan18HHixIk588wzM3jw4HTv3j1TpkzJVlttlZtuummd7W+77bacffbZ6dWrV7p27Zobb7wxVVVVmTFjxhauHAAAAADYkFoNH1etWpVnn302Bx10UOlYvXr1ctBBB+XJJ5/cpDFWrFiR1atXp3Xr1uv8fuXKlXn77berbQAAAABA8Wo1fPzrX/+aysrKtGvXrtrxdu3a5Y033tikMS688MJ06NChWoD5YePHj0/Lli1L2w477PAv1w0AAAAAbFytP3b9r5gwYUJ+/vOf5+67706TJk3W2eaiiy7K0qVLS9vrr7++hasEAAAAgE+nBrV58jZt2qR+/fpZvHhxteOLFy9O+/btN9j3+9//fiZMmJDp06enZ8+e623XuHHjNG7cuEbqBQAAAAA2Xa3OfGzUqFF23333aovFfLB4zD777LPefv/xH/+Ryy+/PA899FD69OmzJUoFAAAAADZTrc58TJLhw4fn1FNPTZ8+fbLnnntm8uTJWb58eQYPHpwkOeWUU7L99ttn/PjxSZIrr7wyo0ePzk9/+tN06tSp9G7I5s2bp3nz5rV2HQAAAABAdbUePn71q1/NX/7yl4wePTpvvPFGevXqlYceeqi0CM3ChQtTr97/TdC87rrrsmrVqhx77LHVxrnkkkty6aWXbsnSAQAAAIANqPXwMUmGDBmSIUOGrPO7ioqKavuvvfZa8QUBAAAAAP+yT/Rq1wAAAADAx5fwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACiE8BEAAAAAKITwEQAAAAAohPARAAAAACjExyJ8/OEPf5hOnTqlSZMm2WuvvfLUU09tsP1///d/p2vXrmnSpEl69OiRBx54YAtVCgAAAABsqloPH2+//fYMHz48l1xySX7zm99k1113TXl5eZYsWbLO9k888UROOOGEnH766ZkzZ06OPvroHH300Xn++ee3cOUAAAAAwIbUevg4ceLEnHnmmRk8eHC6d++eKVOmZKuttspNN920zvY/+MEPcuihh+aCCy5It27dcvnll6d379659tprt3DlAAAAAMCGNKjNk69atSrPPvtsLrrootKxevXq5aCDDsqTTz65zj5PPvlkhg8fXu1YeXl57rnnnnW2X7lyZVauXFnaX7p0aZLk7bff/sh1V678x0fuuy7vNKys0fGS5L1/vFfjYy6v4SH/sXJFzQ6Y5N3Vq2t8zH/lXvlXuddqhntt49xrNcO9tnHutZrhXts491rNcK9tnHutZrjXNs69VjPcaxvnXqsZdf1e+6DfmjVrNt54TS3605/+tCbJmieeeKLa8QsuuGDNnnvuuc4+DRs2XPPTn/602rEf/vCHa9q2bbvO9pdccsmaJDabzWaz2Ww2m81ms9lsNputBrfXX399o/lfrc583BIuuuiiajMlq6qq8ve//z2f+cxnUlZWVouVfbK8/fbb2WGHHfL6669n6623ru1yqMPca2wp7jW2FPcaW4p7jS3FvcaW4l5jS3Gvbb41a9bknXfeSYcOHTbatlbDxzZt2qR+/fpZvHhxteOLFy9O+/bt19mnffv2m9W+cePGady4cbVjrVq1+uhFf8ptvfXW/oPIFuFeY0txr7GluNfYUtxrbCnuNbYU9xpbintt87Rs2XKT2tXqgjONGjXK7rvvnhkzZpSOVVVVZcaMGdlnn33W2Wefffap1j5JHn300fW2BwAAAABqR60/dj18+PCceuqp6dOnT/bcc89Mnjw5y5cvz+DBg5Mkp5xySrbffvuMHz8+STJ06ND069cvV111VY444oj8/Oc/zzPPPJMf//jHtXkZAAAAAMA/qfXw8atf/Wr+8pe/ZPTo0XnjjTfSq1evPPTQQ2nXrl2SZOHChalX7/8maPbt2zc//elP893vfjff+c53sssuu+See+7JF7/4xdq6hE+Fxo0b55JLLlnrEXaoae41thT3GluKe40txb3GluJeY0txr7GluNeKVbZmzaasiQ0AAAAAsHlq9Z2PAAAAAEDdJXwEAAAAAAohfAQAAAAACiF8BAAAAAAKIXxkg379619nwIAB6dChQ8rKynLPPffUdknUQePHj88ee+yRFi1apG3btjn66KMzf/782i6LOui6665Lz549s/XWW2frrbfOPvvskwcffLC2y+JTYMKECSkrK8uwYcNquxTqmEsvvTRlZWXVtq5du9Z2WdRRf/rTn3LyySfnM5/5TJo2bZoePXrkmWeeqe2yqIM6deq01n+3lZWV5Zxzzqnt0qhDKisrM2rUqHTu3DlNmzbN5z73uVx++eWxLnPNa1DbBfDxtnz58uy666457bTTcswxx9R2OdRRM2fOzDnnnJM99tgj7733Xr7zne/kkEMOye9+97s0a9astsujDvnsZz+bCRMmZJdddsmaNWtyyy235KijjsqcOXPyhS98obbLo456+umnc/3116dnz561XQp11Be+8IVMnz69tN+ggX/Fp+a9+eab2XffffOlL30pDz74YLbddtv84Q9/yDbbbFPbpVEHPf3006msrCztP//88zn44INz3HHH1WJV1DVXXnllrrvuutxyyy35whe+kGeeeSaDBw9Oy5Yt861vfau2y6tT/JsJG3TYYYflsMMOq+0yqOMeeuihavtTp05N27Zt8+yzz+aAAw6opaqoiwYMGFBtf+zYsbnuuuvyP//zP8JHCrFs2bKcdNJJueGGG3LFFVfUdjnUUQ0aNEj79u1ruwzquCuvvDI77LBDbr755tKxzp0712JF1GXbbrtttf0JEybkc5/7XPr161dLFVEXPfHEEznqqKNyxBFHJHl/xu3PfvazPPXUU7VcWd3jsWvgY2fp0qVJktatW9dyJdRllZWV+fnPf57ly5dnn332qe1yqKPOOeecHHHEETnooINquxTqsD/84Q/p0KFDdtppp5x00klZuHBhbZdEHXTvvfemT58+Oe6449K2bdvstttuueGGG2q7LD4FVq1alZ/85Cc57bTTUlZWVtvlUIf07ds3M2bMyEsvvZQkmTdvXmbNmmUCVgHMfAQ+VqqqqjJs2LDsu++++eIXv1jb5VAH/fa3v80+++yTd999N82bN8/dd9+d7t2713ZZ1EE///nP85vf/CZPP/10bZdCHbbXXntl6tSp6dKlSxYtWpQxY8Zk//33z/PPP58WLVrUdnnUIa+88kquu+66DB8+PN/5znfy9NNP51vf+lYaNWqUU089tbbLow6755578tZbb2XQoEG1XQp1zMiRI/P222+na9euqV+/fiorKzN27NicdNJJtV1anSN8BD5WzjnnnDz//POZNWtWbZdCHdWlS5fMnTs3S5cuzZ133plTTz01M2fOFEBSo15//fUMHTo0jz76aJo0aVLb5VCHfXh2Rs+ePbPXXntlxx13zB133JHTTz+9FiujrqmqqkqfPn0ybty4JMluu+2W559/PlOmTBE+Uqj//M//zGGHHZYOHTrUdinUMXfccUduu+22/PSnP80XvvCFzJ07N8OGDUuHDh3891oNEz4CHxtDhgzJfffdl1//+tf57Gc/W9vlUEc1atQoO++8c5Jk9913z9NPP50f/OAHuf7662u5MuqSZ599NkuWLEnv3r1LxyorK/PrX/861157bVauXJn69evXYoXUVa1atcrnP//5vPzyy7VdCnXMdtttt9b/UdetW7f84he/qKWK+DT44x//mOnTp+euu+6q7VKogy644IKMHDkyX/va15IkPXr0yB//+MeMHz9e+FjDhI9ArVuzZk3OPffc3H333amoqPDycraoqqqqrFy5srbLoI758pe/nN/+9rfVjg0ePDhdu3bNhRdeKHikMMuWLcuCBQvy9a9/vbZLoY7Zd999M3/+/GrHXnrppey44461VBGfBjfffHPatm1bWhAEatKKFStSr171pVDq16+fqqqqWqqo7hI+skHLli2r9v+cv/rqq5k7d25at26djh071mJl1CXnnHNOfvrTn2batGlp0aJF3njjjSRJy5Yt07Rp01qujrrkoosuymGHHZaOHTvmnXfeyU9/+tNUVFTk4Ycfru3SqGNatGix1ntrmzVrls985jPeZ0uNGjFiRAYMGJAdd9wxf/7zn3PJJZekfv36OeGEE2q7NOqY8847L3379s24ceNy/PHH56mnnsqPf/zj/PjHP67t0qijqqqqcvPNN+fUU09NgwaiC2regAEDMnbs2HTs2DFf+MIXMmfOnEycODGnnXZabZdW55StWbNmTW0XwcdXRUVFvvSlL611/NRTT83UqVO3fEHUSetbte7mm2/2Ymlq1Omnn54ZM2Zk0aJFadmyZXr27JkLL7wwBx98cG2XxqdA//7906tXr0yePLm2S6EO+drXvpZf//rX+dvf/pZtt902++23X8aOHZvPfe5ztV0addB9992Xiy66KH/4wx/SuXPnDB8+PGeeeWZtl0Ud9cgjj6S8vDzz58/P5z//+douhzronXfeyahRo3L33XdnyZIl6dChQ0444YSMHj06jRo1qu3y6hThIwAAAABQiHobbwIAAAAAsPmEjwAAAABAIYSPAAAAAEAhhI8AAAAAQCGEjwAAAABAIYSPAAAAAEAhhI8AAAAAQCGEjwAAfOJ16tQpkydPru0yAAD4J8JHAABqVP/+/TNs2LDaLgMAgI8B4SMAAAAAUAjhIwAANWbQoEGZOXNmfvCDH6SsrCxlZWV57bXXMnPmzOy5555p3Lhxtttuu4wcOTLvvfdeqV///v0zZMiQDBkyJC1btkybNm0yatSorFmz5iPVceONN6ZVq1aZMWNGTV0aAAAfgfARAIAa84Mf/CD77LNP/r/27t8lqz2OA/j76VGHUAQnhwSHJNMhFyUtIWjzj9AhbAhEEMOgUARxiMrCoKEaokFwa0jBTSzStnAocVAqdWkMnknuHe5FrkOg9z7PTeT1grN8z4/P97O+z/d7zuDgYPb29rK3t5fq6ur09fWls7Mznz59yrNnz/Ly5ctMTU0duvfVq1epqqrKx48f8+TJkzx69CgvXrw49hzu37+fO3fuZGlpKdevXy9XawAA/AtVv3sCAACcHvX19ampqcnZs2fT2NiYJLl7926ampry9OnTFAqFtLa2Znd3N2NjYxkfH8+ZM3+9D29qasrMzEwKhUIuXLiQ9fX1zMzMZHBw8Mj1x8bG8vr16ywvL6e9vb0iPQIAcHRWPgIAUFGfP39Od3d3CoXCwdiVK1fy8+fPfP/+/WDs8uXLh67p7u7O5uZm9vf3j1Tn4cOHef78ed69eyd4BAA4IYSPAACcCr29vdnf38/8/PzvngoAAH8TPgIAUFY1NTWHVitevHgxHz58OPTzmPfv36euri7nzp07GFtbWzv0nNXV1bS0tKRYLB6pbldXVxYXFzM9PZ0HDx78xy4AACgH4SMAAGXV3NyctbW1bG9v58ePH7l161a+ffuWoaGhfPnyJW/evMnExERGRkYOvveYJF+/fs3IyEg2NjYyNzeX2dnZDA8PH6t2T09PFhYWMjk5mcePH5e5MwAAjssPZwAAKKvR0dEMDAykra0tpVIpW1tbWVhYyO3bt3Pp0qU0NDTkxo0buXfv3qH7+vv7UyqV0tXVlWKxmOHh4dy8efPY9a9evZq3b9+mr68vxWIxQ0ND5WoNAIBjKvzxz/0vAADwG1y7di0dHR1WKwIAnDK2XQMAAAAAFSF8BADgRFtZWUltbe0vDwAATi7brgEAONFKpVJ2dnZ+ef78+fP/42wAADgO4SMAAAAAUBG2XQMAAAAAFSF8BAAAAAAqQvgIAAAAAFSE8BEAAAAAqAjhIwAAAABQEcJHAAAAAKAihI8AAAAAQEUIHwEAAACAivgTbsqZ0mQuU6IAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1600x800 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "plt.figure(figsize=(16, 8))\n",
    "sns.barplot(x='top_k', y='hit', hue='reranker', data=hit_stat_df, errorbar=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d1fef3ac-4928-48ef-bf73-7d3ee33e9d19",
   "metadata": {},
   "source": [
    "# 预测"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "72108fa6-06ac-4873-a73e-300a087d751e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T08:09:14.248812Z",
     "iopub.status.busy": "2024-09-02T08:09:14.248685Z",
     "iopub.status.idle": "2024-09-02T08:09:14.258403Z",
     "shell.execute_reply": "2024-09-02T08:09:14.257977Z",
     "shell.execute_reply.started": "2024-09-02T08:09:14.248798Z"
    }
   },
   "outputs": [],
   "source": [
    "from langchain.llms import Ollama\n",
    "\n",
    "llm = Ollama(\n",
    "    model='qwen2:7b-instruct',\n",
    "    base_url='http://localhost:11434',\n",
    "    temperature=0\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "c40bb5d6-9b9c-42c0-b12d-e33522186229",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T08:09:14.259182Z",
     "iopub.status.busy": "2024-09-02T08:09:14.258879Z",
     "iopub.status.idle": "2024-09-02T08:09:14.265596Z",
     "shell.execute_reply": "2024-09-02T08:09:14.265166Z",
     "shell.execute_reply.started": "2024-09-02T08:09:14.259169Z"
    }
   },
   "outputs": [],
   "source": [
    "def rag(retriever, reranker, query, n_chunks=4, retirever_multiplier=3):\n",
    "    prompt_tmpl = \"\"\"\n",
    "你是一个金融分析师，擅长根据所获取的信息片段，对问题进行分析和推理。\n",
    "你的任务是根据所获取的信息片段（<<<<context>>><<<</context>>>之间的内容）回答问题。\n",
    "回答保持简洁，不必重复问题，不要添加描述性解释和与答案无关的任何内容。\n",
    "已知信息：\n",
    "<<<<context>>>\n",
    "{{knowledge}}\n",
    "<<<</context>>>\n",
    "\n",
    "问题：{{query}}\n",
    "请回答：\n",
    "\"\"\".strip()\n",
    "\n",
    "    chunks = retriever.invoke(query)[:n_chunks * retirever_multiplier]\n",
    "    chunks = rerank(reranker, query, chunks, top_k=n_chunks)\n",
    "    prompt = prompt_tmpl.replace('{{knowledge}}', '\\n\\n'.join([doc.page_content for doc in chunks])).replace('{{query}}', query)\n",
    "\n",
    "    return llm(prompt), chunks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "24edd1a1-be8d-4d52-8ab4-a285d923e257",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T08:09:14.266191Z",
     "iopub.status.busy": "2024-09-02T08:09:14.266071Z",
     "iopub.status.idle": "2024-09-02T08:09:14.272871Z",
     "shell.execute_reply": "2024-09-02T08:09:14.272439Z",
     "shell.execute_reply.started": "2024-09-02T08:09:14.266179Z"
    }
   },
   "outputs": [],
   "source": [
    "prediction_df = qa_df[qa_df['dataset'] == 'test'][['uuid', 'question', 'qa_type', 'answer']].rename(columns={'answer': 'ref_answer'})\n",
    "\n",
    "def predict(prediction_df, retriever, reranker, n_chunks):\n",
    "    prediction_df = prediction_df.copy()\n",
    "    answer_dict = {}\n",
    "\n",
    "    for idx, row in tqdm(prediction_df.iterrows(), total=len(prediction_df)):\n",
    "        uuid = row['uuid']\n",
    "        question = row['question']\n",
    "        answer, chunks = rag(retriever, reranker, question, n_chunks=n_chunks)\n",
    "        assert len(chunks) <= n_chunks\n",
    "        answer_dict[question] = {\n",
    "            'uuid': uuid,\n",
    "            'ref_answer': row['ref_answer'],\n",
    "            'gen_answer': answer,\n",
    "            'chunks': chunks\n",
    "        }\n",
    "    prediction_df.loc[:, 'gen_answer'] = prediction_df['question'].apply(lambda q: answer_dict[q]['gen_answer'])\n",
    "    prediction_df.loc[:, 'chunks'] = prediction_df['question'].apply(lambda q: answer_dict[q]['chunks'])\n",
    "\n",
    "    return prediction_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "85765906-177d-43a3-a826-82153dbc6deb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T08:09:14.273416Z",
     "iopub.status.busy": "2024-09-02T08:09:14.273295Z",
     "iopub.status.idle": "2024-09-02T08:09:14.279478Z",
     "shell.execute_reply": "2024-09-02T08:09:14.279161Z",
     "shell.execute_reply.started": "2024-09-02T08:09:14.273404Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: http_proxy=\n",
      "env: https_proxy=\n"
     ]
    }
   ],
   "source": [
    "%env http_proxy=\n",
    "%env https_proxy="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "db571983-7054-4f8f-938a-e698dd033797",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T08:09:14.279952Z",
     "iopub.status.busy": "2024-09-02T08:09:14.279832Z",
     "iopub.status.idle": "2024-09-02T08:09:14.285111Z",
     "shell.execute_reply": "2024-09-02T08:09:14.284691Z",
     "shell.execute_reply.started": "2024-09-02T08:09:14.279940Z"
    }
   },
   "outputs": [],
   "source": [
    "pred_dict = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "81881a78-73b0-4aa1-b4d4-e9a16a2571be",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T08:22:24.660998Z",
     "iopub.status.busy": "2024-09-02T08:22:24.660819Z",
     "iopub.status.idle": "2024-09-02T08:36:02.870806Z",
     "shell.execute_reply": "2024-09-02T08:36:02.870187Z",
     "shell.execute_reply.started": "2024-09-02T08:22:24.660984Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ab375a5280a04640b752b6e2b34db7f6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "be010069a07d494c9148a26d4cd6e87c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "flash_attn is not installed. Using PyTorch native attention implementation.\n",
      "flash_attn is not installed. Using PyTorch native attention implementation.\n",
      "flash_attn is not installed. Using PyTorch native attention implementation.\n",
      "flash_attn is not installed. Using PyTorch native attention implementation.\n",
      "flash_attn is not installed. Using PyTorch native attention implementation.\n",
      "flash_attn is not installed. Using PyTorch native attention implementation.\n",
      "flash_attn is not installed. Using PyTorch native attention implementation.\n",
      "flash_attn is not installed. Using PyTorch native attention implementation.\n",
      "flash_attn is not installed. Using PyTorch native attention implementation.\n",
      "flash_attn is not installed. Using PyTorch native attention implementation.\n",
      "flash_attn is not installed. Using PyTorch native attention implementation.\n",
      "flash_attn is not installed. Using PyTorch native attention implementation.\n",
      "flash_attn is not installed. Using PyTorch native attention implementation.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "822b3c640afd496681c3113de6a0b426",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "n_chunks = 3\n",
    "\n",
    "get_retriever_fn = build_get_ensemble_retriver_fn(weights=[0.5, 0.5])\n",
    "retriever = get_retriever_fn(n_chunks)\n",
    "\n",
    "for reranker_name, reranker_path in reranker_dict.items():\n",
    "    # API用量用完了，先跳过\n",
    "    if reranker_name == 'cohere-rerank':\n",
    "        continue\n",
    "\n",
    "    if reranker_name in pred_dict:\n",
    "        continue\n",
    "\n",
    "    model_path = reranker_dict[reranker_name]\n",
    "    reranker = get_reranker(model_path)\n",
    "    pred_df = predict(prediction_df, retriever, reranker, n_chunks=n_chunks)\n",
    "\n",
    "    pred_dict[reranker_name] = pred_df\n",
    "    \n",
    "    # 释放显存\n",
    "    torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c88911a7-71da-42b8-aaba-348a3131d657",
   "metadata": {},
   "source": [
    "# 评估 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "e2343b64-57a1-4a77-a25d-a769ea018692",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T09:32:22.539674Z",
     "iopub.status.busy": "2024-09-02T09:32:22.538868Z",
     "iopub.status.idle": "2024-09-02T09:32:22.555573Z",
     "shell.execute_reply": "2024-09-02T09:32:22.555069Z",
     "shell.execute_reply.started": "2024-09-02T09:32:22.539603Z"
    }
   },
   "outputs": [],
   "source": [
    "from langchain_openai import ChatOpenAI\n",
    "\n",
    "judge_llm = ChatOpenAI(\n",
    "    api_key=os.environ['LLM_API_KEY'],\n",
    "    base_url=os.environ['LLM_BASE_URL'],\n",
    "    model_name='qwen2-72b-instruct',\n",
    "    temperature=0\n",
    ")\n",
    "\n",
    "import time\n",
    "\n",
    "def evaluate(prediction_df):\n",
    "    \"\"\"\n",
    "    对预测结果进行打分\n",
    "    :param prediction_df: 预测结果，需要包含问题，参考答案，生成的答案，列名分别为question, ref_answer, gen_answer\n",
    "    :return 打分模型原始返回结果\n",
    "    \"\"\"\n",
    "    prompt_tmpl = \"\"\"\n",
    "你是一个经济学博士，现在我有一系列问题，有一个助手已经对这些问题进行了回答，你需要参照参考答案，评价这个助手的回答是否正确，仅回复“是”或“否”即可，不要带其他描述性内容或无关信息。\n",
    "问题：\n",
    "<question>\n",
    "{{question}}\n",
    "</question>\n",
    "\n",
    "参考答案：\n",
    "<ref_answer>\n",
    "{{ref_answer}}\n",
    "</ref_answer>\n",
    "\n",
    "助手回答：\n",
    "<gen_answer>\n",
    "{{gen_answer}}\n",
    "</gen_answer>\n",
    "请评价：\n",
    "    \"\"\"\n",
    "    results = []\n",
    "\n",
    "    for _, row in tqdm(prediction_df.iterrows(), total=len(prediction_df)):\n",
    "        question = row['question']\n",
    "        ref_answer = row['ref_answer']\n",
    "        gen_answer = row['gen_answer']\n",
    "\n",
    "        prompt = prompt_tmpl.replace('{{question}}', question).replace('{{ref_answer}}', str(ref_answer)).replace('{{gen_answer}}', gen_answer).strip()\n",
    "\n",
    "        retry_count = 3\n",
    "        result = None\n",
    "        \n",
    "        while retry_count > 0:\n",
    "            try:\n",
    "                result = judge_llm.invoke(prompt).content\n",
    "                break\n",
    "            except:\n",
    "                time.sleep(4 ** (5 - retry_count))\n",
    "                retry_count -= 1\n",
    "\n",
    "        if result is None:\n",
    "            print(f'question={question}, scoring failed!')\n",
    "        results.append(result)\n",
    "        \n",
    "        time.sleep(1)\n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "edda3e5b-02bd-432f-bf93-a4fb6cb8568c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T09:32:23.549501Z",
     "iopub.status.busy": "2024-09-02T09:32:23.548729Z",
     "iopub.status.idle": "2024-09-02T09:45:27.684338Z",
     "shell.execute_reply": "2024-09-02T09:45:27.681904Z",
     "shell.execute_reply.started": "2024-09-02T09:32:23.549430Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a20636d640a844cabcd911ac05ef3535",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bge-reranker-base: ['是' '否']\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "39d11050a024421d9604a7b7b64f6957",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "base-reranker-large: ['是' '否']\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "84595c4d349944c6b32c2432689bcf11",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bge-reranker-v2-m3: ['是' '否']\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a6adf3053d2f4465936ba9841c908f4c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gte-reranker: ['是' '否']\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b3690336ffe4490e8da9a05b6deeba8c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "jina-reranker: ['是' '否']\n"
     ]
    }
   ],
   "source": [
    "for reranker_name in pred_dict:\n",
    "    pred_df = pred_dict[reranker_name]\n",
    "    pred_df['raw_score'] = evaluate(pred_df)\n",
    "    print(f\"{reranker_name}: {pred_df['raw_score'].unique()}\")\n",
    "    pred_df['score'] = (pred_df['raw_score'] == '是').astype(int)\n",
    "    \n",
    "    time.sleep(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "96630464-0957-474b-94ca-21fcc31386d8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T09:45:29.604696Z",
     "iopub.status.busy": "2024-09-02T09:45:29.603912Z",
     "iopub.status.idle": "2024-09-02T09:45:29.610056Z",
     "shell.execute_reply": "2024-09-02T09:45:29.609668Z",
     "shell.execute_reply.started": "2024-09-02T09:45:29.604626Z"
    }
   },
   "outputs": [],
   "source": [
    "metrics = []\n",
    "\n",
    "for reranker_name in pred_dict:\n",
    "    pred_df = pred_dict[reranker_name]\n",
    "    metrics.append({\n",
    "        'reranker': reranker_name,\n",
    "        'score': pred_df['score'].mean()\n",
    "    })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "c72cf8a9-ac69-4a70-928b-61a109eefe0f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T09:45:30.356610Z",
     "iopub.status.busy": "2024-09-02T09:45:30.356396Z",
     "iopub.status.idle": "2024-09-02T09:45:30.362983Z",
     "shell.execute_reply": "2024-09-02T09:45:30.362398Z",
     "shell.execute_reply.started": "2024-09-02T09:45:30.356593Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>reranker</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>bge-reranker-base</td>\n",
       "      <td>0.80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>base-reranker-large</td>\n",
       "      <td>0.84</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>bge-reranker-v2-m3</td>\n",
       "      <td>0.81</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>gte-reranker</td>\n",
       "      <td>0.87</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>jina-reranker</td>\n",
       "      <td>0.80</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              reranker  score\n",
       "0    bge-reranker-base   0.80\n",
       "1  base-reranker-large   0.84\n",
       "2   bge-reranker-v2-m3   0.81\n",
       "3         gte-reranker   0.87\n",
       "4        jina-reranker   0.80"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(metrics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6611cc71-398a-4a9f-9c7d-c80f5cb1b88c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
